Re: [HACKERS] Transactions involving multiple postgres foreign servers, take 2

From 7e9ced2ad44e5bc1dd651f18083b948daadbe7b8 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Thu, 8 Feb 2018 11:26:46 +0900
Subject: [PATCH v17 1/4] Keep track of writing on non-temporary relation.

---
 src/backend/access/heap/heapam.c | 12 ++++++++++++
 src/include/access/xact.h        |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 72395a5..959a331 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2611,6 +2611,10 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 		heap_freetuple(heaptup);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleGetOid(tup);
 }
 
@@ -3440,6 +3444,10 @@ l1:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleMayBeUpdated;
 }
 
@@ -4390,6 +4398,10 @@ l2:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	bms_free(hot_attrs);
 	bms_free(proj_idx_attrs);
 	bms_free(key_attrs);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 083e879..c7b4144 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -98,6 +98,11 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
-- 
2.10.5

From f879e1d0b31728b9a15683ba06ceb9badfa9d56f Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:44:42 +0900
Subject: [PATCH v17 2/4] Support atomic commit among multiple foreign servers.

---
 doc/src/sgml/catalogs.sgml                    |   97 +
 doc/src/sgml/config.sgml                      |  124 ++
 doc/src/sgml/fdwhandler.sgml                  |  200 ++
 doc/src/sgml/func.sgml                        |   51 +
 doc/src/sgml/monitoring.sgml                  |   56 +
 src/backend/access/rmgrdesc/Makefile          |    8 +-
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   65 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/Makefile           |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   32 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/foreigncmds.c            |   23 +
 src/backend/executor/execPartition.c          |    4 +
 src/backend/executor/nodeForeignscan.c        |    8 +
 src/backend/executor/nodeModifyTable.c        |    5 +
 src/backend/foreign/Makefile                  |    2 +-
 src/backend/foreign/fdwxact.c                 | 2762 +++++++++++++++++++++++++
 src/backend/foreign/fdwxact_launcher.c        |  587 ++++++
 src/backend/foreign/fdwxact_resolver.c        |  310 +++
 src/backend/foreign/foreign.c                 |   43 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   18 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    5 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    2 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   61 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   23 +
 src/include/foreign/fdwapi.h                  |   18 +-
 src/include/foreign/fdwxact.h                 |  147 ++
 src/include/foreign/fdwxact_launcher.h        |   31 +
 src/include/foreign/fdwxact_resolver.h        |   23 +
 src/include/foreign/fdwxact_xlog.h            |   51 +
 src/include/foreign/foreign.h                 |    2 +-
 src/include/foreign/resolver_internal.h       |   65 +
 src/include/pgstat.h                          |    8 +-
 src/include/storage/proc.h                    |   10 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    2 +
 src/test/regress/expected/rules.out           |   12 +
 57 files changed, 5052 insertions(+), 27 deletions(-)
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 100755 src/backend/foreign/fdwxact.c
 create mode 100644 src/backend/foreign/fdwxact_launcher.c
 create mode 100644 src/backend/foreign/fdwxact_resolver.c
 create mode 100644 src/include/foreign/fdwxact.h
 create mode 100644 src/include/foreign/fdwxact_launcher.h
 create mode 100644 src/include/foreign/fdwxact_resolver.h
 create mode 100644 src/include/foreign/fdwxact_xlog.h
 create mode 100644 src/include/foreign/resolver_internal.h

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index fffb79f..e0d8157 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9629,6 +9629,103 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-prepared-fdw-xacts">
+  <title><structname>pg_prepared_fdw_xacts</structname></title>
+
+  <indexterm zone="view-pg-prepared-fdw-xacts">
+   <primary>pg_prepared_fdw_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_prepared_fdw_xacts</structname> displays
+   information about foreign transactions that are currently prepared on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="fdw-transaction-managements"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_prepared_xacts</structname> contains one row per prepared
+   foreign transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_prepared_fdw_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>transaction</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Transaction id that this foreign transaction associates with
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server that this foreign server is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction: <literal>prepared</literal>, <literal>committing</literal>, <literal>aborting</literal> or <literal>unknown</literal>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index bee4afb..044cc7b 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1546,6 +1546,29 @@ include_dir 'conf.d'
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+      <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Sets the maximum number of foreign transactions that can be prepared
+        simultaneously. A single local transaction can give rise to multiple
+        foreign transaction. If <literal>N</literal> local transactions each
+        across <literal>K</literal> foreign server this value need to be set
+        <literal>N * K</literal>, not just <literal>N</literal>.
+        This parameter can only be set at server start.
+       </para>
+       <para>
+        When running a standby server, you must set this parameter to the
+        same or higher value than on the master server. Otherwise, queries
+        will not be allowed in the standby server.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-work-mem" xreflabel="work_mem">
       <term><varname>work_mem</varname> (<type>integer</type>)
       <indexterm>
@@ -3607,6 +3630,78 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
      </variablelist>
     </sect2>
 
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+
+     <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+      <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+      <indexterm>
+       <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+        resolver is responsible for foreign transaction resolution on one database.
+       </para>
+       <para>
+        Foreign transaction resolution workers are taken from the pool defined by
+        <varname>max_worker_processes</varname>.
+       </para>
+       <para>
+        The default value is 0.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry id="foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+      <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specify how long the foreign transaction resolver should wait when the last resolution
+        fails before retrying to resolve foreign transaction. This parameter can only be set in the
+        <filename>postgresql.conf</filename> file or on the server command line.
+       </para>
+       <para>
+        The default value is 10 seconds.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry id="foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+      <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Terminate foreign transaction resolver processes that don't have any foreign
+        transactions to resolve longer than the specified number of milliseconds.
+        A value of zero disables the timeout mechanism.  You should set this value to
+        zero only if you set <varname>max_foreign_transaction_resolvers</varname> as
+        much as databases you have. This parameter can only be set in the
+        <filename>postgresql.conf</filename> file or on the server command line.
+       </para>
+       <para>
+        The default value is 60 seconds.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     </variablelist>
+    </sect2>
+
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -7822,6 +7917,35 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-foreign-transaction">
+    <title>Foreign Transaction Management</title>
+
+    <variablelist>
+
+     <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophase_commit">
+      <term><varname>foreign_twophase_commit</varname> (<type>bool</type>)
+       <indexterm>
+        <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+       </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies whether transaction commit will wait for all involving foreign transaction
+        to be resolved before the command returns a "success" indication to the client.
+        Both <varname>max_prepared_foreign_transactions</varname> and
+        <varname>max_foreign_transaction_resolvers</varname> must be non-zero value to
+        allow foreign twophase commit to be used.
+       </para>
+       <para>
+        This parameter can be changed at any time; the behavior for any one transaction
+        is determined by the setting in effect when it commits.
+       </para>
+      </listitem>
+     </varlistentry>
+    </variablelist>
+
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 4ce88dd..24c635c 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1390,6 +1390,109 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     If an FDW wishes to support <firstterm>atomic commit</firstterm>
+     (as described in <xref linkend="fdw-transaction-managements"/>), it must call the
+     registrasaction function <function>FdwXactRegisterForeignTransaction</function>
+     and provide the following callback functions:
+    </para>
+
+    <para>
+<programlisting>
+bool
+PrepareForeignTransaction(ForeignTransaction *foreign_xact);
+</programlisting>
+    Prepare a foreign transaction identified by <varname>foreign_xact</varname>.
+    This function is called at the pre-commit phase of the local
+    transaction if atomic commit is
+    required. Returning <literal>true</literal> means that preparing
+    the foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(ForeignTransaction *foreign_xact);
+</programlisting>
+    Commit a not-prepared foreign transaction identified by
+    <varname>foreign_xact</varname>.
+    This function is called at the pre-commit phase of local
+    transaction if atomic commit is not required. The atomic
+    commit is not required either when we modified data on
+    only one server including local server or when user doesn't
+    request atomic commit by <xref linkend="guc-foreign-twophase-commit"/>.
+    Returning <literal>true</literal> means that commit the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(ForeignTransaction *foreign_xact);
+</programlisting>
+    Rollback a not-prepared foreign transaction identified by
+    <varname>foreign_xact</varname>.
+    This function is called at the end of local transaction after
+    rollbacked locally either when user requested rollback or when
+    any error occurs within the transaction. This function could
+    be called recursively if any error occurs during rollback the
+    foreign transaction for whatever reason. You need to track
+    recursion and prevent this function from being called infinitely.
+    Returning <literal>true</literal> means that rollback the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+ResolvePreparedForeignTransaction(ForeignTransaction *foreign_xact,
+                                  bool is_commit);
+</programlisting>
+    Commit or rollback the prepared foreign transaction identified
+    by <varname>foreign_xact</varname>. on a connection to foreign server
+    When <varname>is_commit</varname> is true, it indicate that the foreign
+    transaction should be committed.
+    This function normally is called by the foreign transaction resolver
+    process but can also be called by <function>pg_resovle_fdw_xacts</function>
+    function. In the resolver process, this function is called either
+    when a backend requests the resolver process to resolve a distributed
+    transaction after prepared or when a database has dangling
+    transaction. Returning <literal>true</literal> means that resolving
+    the foreign transaction got successful.
+    In abort case, please note that the prepared foreign transaction
+    having identifier <varname>foreign__xact->fx_id</varname> might not
+    exist on the foreign server. If you failed to resolve the foreign
+    transaction due to undefined object error
+    (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) you should regards
+    it as success and return <literal>true</literal>.
+    </para>
+    <para>
+<programlisting>
+bool
+IsTwoPhaseCommitEnabled(Oid serverid);
+</programlisting>
+    Return <literal>true</literal> if foreign server identified by
+    <literal>serverid</literal> is capable of two-phase commit protocol.
+    This function is called when the transaction begins to modify data on
+    the foreign server. Return <literal>false</literal> indicates that
+    the current transaction cannot use atomic commit even if atomic commit
+    is requested by user.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>. To get informations of FDW-related
+      objects, you can use given a <literal>ForeignTransaction</literal>
+      instead (see <filename>foreign/fdwxact.h</filename> for details).
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1835,4 +1938,101 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+    <title>Transaction managements for Foreign Data Wrappers</title>
+
+    <sect2 id="fdw-transaction-atomic-commit">
+     <title>Atomic commit among multiple foreign servers</title>
+
+     <para>
+      <productname>PostgreSQL</productname> foreign transaction manager
+      allows FDWs to read and write data on foreign server within a transaction while
+      maintaining atomicity of the foreign data (aka atomic commit). Using
+      atomic commit, it guarantees that a distributed transaction is committed
+      or rollbacked on all participants foreign
+      server.  To achieve atomic commit, <productname>PostgreSQL</productname>
+      employees two-phase commit protocol, which is a type of atomic commitment
+      protocol. Every FDW that wish to support atomic commit
+      is required to support transaction management callback routines
+      (see <xref linkend="fdw-callbacks-transaction-managements"/> for details)
+      and register the foreign transaction using
+      <function>FdwXactRegisterForeignTransaction</function> when starting a
+      transaction on the foreign server. Transaction of registered foreign server
+      is managed by the foreign transaction manager.
+<programlisting>
+void
+FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, char *fx_id)
+</programlisting>
+    This function should be called when a transaction starts on the foreign server.
+    <varname>serverid</varname> and <varname>userid</varname> are <type>OID</type>s
+    which specify the transaction starts on what server by who. <varname>fx_id</varname>
+    is null-terminated string which is an identifer of foreign transaction and it
+    will be passed when transaction management APIs is called. The length of
+    <varname>fx_id</varname> must be less than 200 bytes. Also this identifier
+    must be unique enough so that it doesn't conflict other concurrent foreign
+    transactions. <varname>fx_id</varname> can be <literal>NULL</literal>.
+    If it's <literal>NULL</literal>, a transaction identifier is automacitally
+    generated with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    Since this identifier is used per foreign transaction and the xid of unresolved
+    distributed transaction never reused, an auto-generated identifier is fairly
+    enough to ensure uniqueness. It's recommended to generate foreign transaction
+    identifier in FDW if the format of auto-generated identifier doesn't match
+    the requirement of the foreign server.
+    </para>
+
+     <para>
+      An example of such transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+     </para>
+
+     <para>
+      When a transaction starts on the foreign server, FDW that wishes atomic
+      commit must register the foreign transaction as a participant by calling
+      <function>FdwXactRegisterForeignTransaction</function>. Also during
+      transaction, <function>IsTwoPhaseCommitEnabled</function> is called whenever
+      the transaction begins to modify data on the foreign server. If FDW wishes
+      atomic commit <function>IsTwoPhaseCommitEnabled</function> must return
+      <literal>true</literal>. All foreign transaction participants must
+      return <literal>true</literal> to achieve atomic commit.
+     </para>
+
+     <para>
+      During pre-commit phase of local transaction, the foreign transaction manager
+      persists the foreign transaction information to the disk and WAL, and then
+      prepare all foreign transaction by calling <function>PrepareForeignTransaction</function>
+      if two-phase commit protocol is required. Two-phase commit is required only if
+      the transaction modified data on more than one servers including the local
+      server and user requests atomic commit. <productname>PostgreSQL</productname>
+      can commit locally and go to the next step if and only if all preparing foreign
+      transactions got successful. If two-phase commit is not required, the foreign
+      transaction manager commits a transaction on the foreign server by calling
+      <function>CommitForeignTransaction</function> and then
+      <productname>PostgreSQL</productname> commits locally. The foreign transaction
+      manager doesn't do any further change on foreign transactions from this point
+      forward. If any failure happens for whatever reason, for example a network
+      failure or user request until <productname>PostgreSQL</productname> commits
+      locally the foreign transaction manager changes over to rollback and calls
+      <function>RollbackForeignTransaction</function> for every foreign servers to
+      close the current transaction on foreign servers.
+     </para>
+
+     <para>
+      When two-phase commit is required, after committed locally, each the transaction
+      commits will wait for all prepared foreign transaction to be resolved before
+      the commit completes. The foreign transaction resolver is responsible for
+      foreign transaction resolution. <function>ResolverForeignTransaction</function>
+      is called by the foreign transaction resolver process when it resolves a foreign
+      transactions. <function>ResolveForeignTransaction</function> is also be called
+      when user execute <function>pg_resovle_fdw_xact</function> function.
+     </para>
+    </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index edc9be9..7b7fe1f 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -20553,6 +20553,57 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-fdw-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_fdw_xacts</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_fdw_xacts</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function search for foreign transaction
+        matching the arguments and resolves then. This function won't resolve
+        a foreign transaction which is in progress, or one that is locked by some
+        other backend.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_fdw_xact</function>
+        except it remove foreign transaction entry without resolving.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 0484cfa..635a5e7 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -332,6 +332,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_fdw_xact_resolver</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-resolver-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1194,6 +1202,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
          <entry><literal>LogicalLauncherMain</literal></entry>
          <entry>Waiting in main loop of logical launcher process.</entry>
         </row>
@@ -1405,6 +1421,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
+        <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
          <entry>Waiting during base backup when throttling activity.</entry>
@@ -2214,6 +2234,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-resolver-view" xreflabel="pg_stat_fdw_xact_resolver">
+   <title><structname>pg_stat_fdw_xact_resolver</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index 5514db1..742e825 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -8,9 +8,9 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
-	   gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
-	   mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
-	   smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdwxactdesc.o \
+	genericdesc.o  gindesc.o gistdesc.o hashdesc.o heapdesc.o \
+	logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \
+	seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000..3705104
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,65 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdw_xactdesc.c
+ *		PostgreSQL distributed transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/backend/access/transam/fdw_xactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "foreign/fdwxact_xlog.h"
+
+void
+fdw_xact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		FdwXactOnDiskData *fdw_insert_xlog = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_insert_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_insert_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_insert_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_insert_xlog->local_xid);
+		/* TODO: This should be really interpreted by each FDW */
+
+		/*
+		 * TODO: we also need to assess whether we want to add this
+		 * information
+		 */
+		appendStringInfo(buf, " foreign transaction info: %s",
+						 fdw_insert_xlog->fdw_xact_id);
+	}
+	else
+	{
+		xl_fdw_xact_remove *fdw_remove_xlog = (xl_fdw_xact_remove *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_remove_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_remove_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_remove_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_remove_xlog->xid);
+	}
+
+}
+
+const char *
+fdw_xact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDW_XACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDW_XACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 00741c7..023a7c5 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -112,14 +112,16 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_prepared_xacts=%d max_locks_per_xact=%d "
 						 "wal_level=%s wal_log_hints=%s "
-						 "track_commit_timestamp=%s",
+						 "track_commit_timestamp=%s "
+						 "max_prepared_foreign_xacts=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_prepared_xacts,
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47..f15c83a 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -12,9 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
-	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
-	xact.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o generic_xlog.o multixact.o \
+	parallel.o rmgr.o slru.o subtrans.o timeline.o transam.o twophase.o \
+	twophase_rmgr.o varsup.o xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9368b56..b5c3502 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -24,6 +24,7 @@
 #include "commands/dbcommands_xlog.h"
 #include "commands/sequence.h"
 #include "commands/tablespace.h"
+#include "foreign/fdwxact.h"
 #include "replication/message.h"
 #include "replication/origin.h"
 #include "storage/standby.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 306861b..4cc01f1 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -89,6 +89,7 @@
 #include "access/xlogreader.h"
 #include "catalog/pg_type.h"
 #include "catalog/storage.h"
+#include "foreign/fdwxact.h"
 #include "funcapi.h"
 #include "miscadmin.h"
 #include "pg_trace.h"
@@ -843,6 +844,35 @@ TwoPhaseGetGXact(TransactionId xid)
 }
 
 /*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
+/*
  * TwoPhaseGetDummyProc
  *		Get the dummy backend ID for prepared transaction specified by XID
  *
@@ -2335,6 +2365,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2394,6 +2430,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 9aa63c8..d4256f4 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -36,6 +36,7 @@
 #include "commands/tablecmds.h"
 #include "commands/trigger.h"
 #include "executor/spi.h"
+#include "foreign/fdwxact.h"
 #include "libpq/be-fsstubs.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -1127,6 +1128,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_twophase_for_ac;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1135,6 +1137,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_twophase_for_ac = ForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1173,12 +1176,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_twophase_for_ac)
 			goto cleanup;
 	}
 	else
@@ -1336,6 +1340,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_twophase_for_ac && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -1974,6 +1986,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2129,6 +2144,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2216,6 +2232,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2404,6 +2422,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2608,6 +2627,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 493f1db..67ddfb5 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -40,6 +40,7 @@
 #include "catalog/pg_control.h"
 #include "catalog/pg_database.h"
 #include "commands/tablespace.h"
+#include "foreign/fdwxact.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "port/atomics.h"
@@ -5246,6 +5247,7 @@ BootStrapXLOG(void)
 	ControlFile->MaxConnections = MaxConnections;
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6333,6 +6335,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -6853,14 +6858,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdw_xact, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -7052,7 +7058,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7558,6 +7567,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7876,6 +7886,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9192,6 +9205,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9625,7 +9639,8 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9657,6 +9672,7 @@ XLogReportParameters(void)
 		ControlFile->MaxConnections = MaxConnections;
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9854,6 +9870,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -10052,6 +10069,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->MaxConnections = xlrec.MaxConnections;
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 7251552..5fa6065 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -291,6 +291,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_prepared_fdw_xacts AS
+       SELECT * FROM pg_prepared_fdw_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
 	l.objoid, l.classoid, l.objsubid,
@@ -773,6 +776,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_fdwxact_resolvers AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_fdwxact_resolver() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index 5c53aee..3a6dff5 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_user_mapping.h"
 #include "commands/defrem.h"
 #include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
 #include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "parser/parse_func.h"
@@ -1093,6 +1094,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1403,6 +1416,16 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
 	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srv->serverid,
+						useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
+	/*
 	 * Do the deletion
 	 */
 	object.classId = UserMappingRelationId;
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index d13be41..f60804c 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -19,6 +19,7 @@
 #include "executor/execPartition.h"
 #include "executor/executor.h"
 #include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -714,7 +715,10 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+		FdwXactMarkForeignTransactionModified(partRelInfo, 0);
+	}
 
 	MemoryContextSwitchTo(oldContext);
 
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index a2a28b7..30a0b66 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,9 +22,11 @@
  */
 #include "postgres.h"
 
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
 #include "utils/memutils.h"
 #include "utils/rel.h"
 
@@ -224,7 +226,13 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+
+		/* Mark this transaction modified data on the foreign server */
+		FdwXactMarkForeignTransactionModified(estate->es_result_relation_info,
+										 eflags);
+	}
 	else
 		fdwroutine->BeginForeignScan(scanstate, eflags);
 
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index d8d89c7..6554b0b 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -44,6 +44,8 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "storage/bufmgr.h"
@@ -2317,6 +2319,9 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 															 fdw_private,
 															 i,
 															 eflags);
+
+			/* Mark this transaction modified data on the foreign server */
+			FdwXactMarkForeignTransactionModified(resultRelInfo, eflags);
 		}
 
 		resultRelInfo++;
diff --git a/src/backend/foreign/Makefile b/src/backend/foreign/Makefile
index 85aa857..4329d3e 100644
--- a/src/backend/foreign/Makefile
+++ b/src/backend/foreign/Makefile
@@ -12,6 +12,6 @@ subdir = src/backend/foreign
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS= foreign.o
+OBJS= foreign.o fdwxact.o fdwxact_launcher.o fdwxact_resolver.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/foreign/fdwxact.c b/src/backend/foreign/fdwxact.c
new file mode 100755
index 0000000..d284861
--- /dev/null
+++ b/src/backend/foreign/fdwxact.c
@@ -0,0 +1,2762 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL distributed transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers atomically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * When a foreign data wrapper starts transaction on a foreign server
+ * that is capable of two-phase commit protocol, it's required to register
+ * the foreign transaction using function FdwXactRegisterTransaction() in order
+ * to participate to a group for atomic commit. Participants are identified
+ * by oid of foreign server and user. When the foreign transaction begins
+ * to modify data it's required to mark it as modified using
+ * FdwXactMarkForeignTransactionModified()
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. After committing or rolling back locally, we
+ * notify the resolver process and tell it to commit or roll back those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish,
+ * and so that we're not trying to locally do work that might fail when an
+ * ERROR after already committed.
+ *
+ * Two-phase commit protocol is required if the transaction modified
+ * two or more servers including itself. In other case, all foreign transactions
+ * are committed during pre-commit.
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. dangling
+ * transaction). Dangling transactions are processed by the resolve process
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * 	* On PREPARE redo we add the foreign transaction to FdwXactCtl->fdw_xacts.
+ *	  We set fdw_xact->inredo to true for such entries.
+ *	* On Checkpoint redo, we iterate through FdwXactCtl->fdw_xacts entries that
+ *	  have set fdw_xact->inredo true and are behind the redo_horizon. We save
+ *    them to disk and then set fdw_xact->ondisk to true.
+ *	* On COMMIT and ABORT we delete the entry from FdwXactCtl->fdw_xacts.
+ *	  If fdw_xact->ondisk is true, we delete the corresponding file from
+ *	  the disk as well.
+ *  * RecoverFdwXacts loads all foreign transaction entries from disk into
+ *    memory at server startup.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/foreign/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/fdwxact_launcher.h"
+#include "foreign/fdwxact_xlog.h"
+#include "foreign/resolver_internal.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Is atomic commit requested by user? */
+#define AtomicCommitRequested() \
+	(foreign_twophase_commit == true && \
+	 max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Structure to bundle the foreign transaction participant */
+typedef struct FdwXactParticipant
+{
+	Oid			serverid;
+	Oid			userid;
+
+	/*
+	 * Pointer to a FdwXact entry in global entry. NULL if
+	 * this foreign transaction is registered but not inserted
+	 * yet.
+	 */
+	FdwXact		fdw_xact;
+	char		*fdw_xact_id;
+
+	/* true if this transaction modified data on the foreign server */
+	bool		modified;
+
+	/*
+	 * This is initialized at foreign transaction registration and
+	 * passed to API functions.
+	 */
+	ForeignTransaction foreign_xact;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact;
+	CommitForeignTransaction_function	commit_foreign_xact;
+	RollbackForeignTransaction_function	rollback_foreign_xact;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit.
+ * This list has only foreign servers that are capable of two-phase
+ * commit protocol.
+ */
+List *FdwXactParticipantsForAC = NIL;
+
+/*
+ * This struct tracks all participants involved with transaction 'xid'.
+ */
+typedef struct FdwXactStateCacheEntry
+{
+	/* Key -- must be first */
+	TransactionId	xid;
+
+	/* List of FdwXacts involved with the xid */
+	FdwXact	participants;
+} FdwXactStateCacheEntry;
+static HTAB	*FdwXactStateCache;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDW_XACTS_DIR "pg_fdw_xact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDW_XACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDW_XACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+static FdwXact FdwXactRegisterFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static bool FdwXactResolveForeignTransaction(FdwXact fdw_xact);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactQueueInsert(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid, bool give_warnings);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							  bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid, Oid serverid,
+								  Oid userid, XLogRecPtr insert_start_lsn,
+								  bool give_warnings);
+static List *get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						   bool need_lock);
+static FdwXact get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								bool need_lock);
+static FdwXact get_all_fdw_xacts(int *length);
+static FdwXact insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							   char *fdw_xact_id);
+static char *generate_fdw_xact_identifier(Oid serverid, Oid userid);
+static void remove_fdw_xact(FdwXact fdw_xact);
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+bool foreign_twophase_commit = false;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * This function aimes to be called by FDW when foreign transaction
+ * starts. The foreign server identified by given server id must
+ * support atomic commit APIs. The foreign transaction is identified
+ * by given identifier 'fdw_xact_id' which can be NULL. If it's NULL,
+ * we construct an unique identifer.
+ *
+ * After registered, foreign transaction of participants are managed
+ * by foreign transaction manager until the end of the distributed
+ * transaction.
+ */
+void
+FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, char *fx_id)
+{
+	FdwXactParticipant	*fdw_part;
+	ListCell   			*lc;
+	ForeignServer 		*foreign_server;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	FdwRoutine			*fdw_routine;
+	MemoryContext		old_context;
+
+	/* Check length of foreign transaction identifier */
+	if (fx_id != NULL && strlen(fx_id) >= NAMEDATALEN)
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifer \"%s\" is too long",
+						fx_id),
+				 errdetail("foreign transaction identifier must be less than %d characters.",
+						   NAMEDATALEN)));
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_xact_resolvers to a nonzero value.")));
+
+	/* Duplication check */
+	foreach(lc, FdwXactParticipantsForAC)
+	{
+		fdw_part = lfirst(lc);
+
+		/* Quick return if there is already registered connection */
+		if (fdw_part->serverid == serverid && fdw_part->userid == userid)
+			ereport(ERROR,
+					(errmsg("attempt to start transction again on server %u user %u",
+							serverid, userid)));
+	}
+
+	/*
+	 * Participants information is needed at the end of a transaction, when
+	 * system cache are not available. so save it in TopTransactionContext
+	 * before hand so that these can live until the end of transaction.
+	 */
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	foreign_server = GetForeignServer(serverid);
+	fdw = GetForeignDataWrapper(foreign_server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+	user_mapping = GetUserMapping(userid, serverid);
+
+	/* Make sure that the FDW has transaction handlers */
+	if (!fdw_routine->PrepareForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function provided for preparing foreign transaction for FDW %s",
+						fdw->fdwname)));
+	if (!fdw_routine->CommitForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to commit a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+	if (!fdw_routine->RollbackForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to rollback a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+
+	/* Generate foreign transaction identifier if not provided */
+	if (fx_id ==  NULL)
+		fx_id = generate_fdw_xact_identifier(serverid, userid);
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->serverid = serverid;
+	fdw_part->userid = userid;
+	fdw_part->fdw_xact_id = fx_id;
+	fdw_part->fdw_xact = NULL;
+	fdw_part->modified = false;	/* by default */
+	fdw_part->foreign_xact.server = foreign_server;
+	fdw_part->foreign_xact.usermapping = user_mapping;
+	fdw_part->foreign_xact.fx_id = fx_id;
+	fdw_part->prepare_foreign_xact = fdw_routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact = fdw_routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact = fdw_routine->RollbackForeignTransaction;
+
+	/* Add this foreign connection to the participants list */
+	FdwXactParticipantsForAC = lappend(FdwXactParticipantsForAC, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_context);
+
+	return;
+}
+
+/*
+ * Remember the registered foreign transaction modified data . This function
+ * is called when the executor begins to modify data on a foreign server
+ * regardless the foreign server is capable of two-phase commit protocol.
+ * Marking it will be used to determine we must use two-phase commit protocol
+ * at commit. This function also checks if the begin modified foreign server
+ * is capable of two-phase commit or not. If it doesn't support, we remember
+ * it.
+ */
+void
+FdwXactMarkForeignTransactionModified(ResultRelInfo *resultRelInfo, int flags)
+{
+	Relation			rel = resultRelInfo->ri_RelationDesc;
+	FdwXactParticipant	*fdw_part;
+	ForeignTable		*ftable;
+	ListCell   			*lc;
+	Oid					userid;
+	Oid					serverid;
+
+	bool found = false;
+
+	/* Quick return if user not request */
+	if (!AtomicCommitRequested())
+		return;
+
+	/* Do nothing in EXPLAIN (no ANALYZE) case */
+	if (flags && EXEC_FLAG_EXPLAIN_ONLY)
+		return;
+
+	ftable = GetForeignTable(RelationGetRelid(rel));
+
+	/*
+	 * If the being modified foreign server doesn't or cannot enable
+	 * two-phase commit protocol, mark that we've written such server
+	 * and return.
+	 */
+	if (resultRelInfo->ri_FdwRoutine->IsTwoPhaseCommitEnabled == NULL ||
+		!resultRelInfo->ri_FdwRoutine->IsTwoPhaseCommitEnabled(ftable->serverid))
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		return;
+	}
+
+	/*
+	 * The foreign server being modified supports two-phase commit protocol,
+	 * remember that the foreign transaction modified data.
+	 */
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	serverid = ftable->serverid;
+	foreach(lc, FdwXactParticipantsForAC)
+	{
+		fdw_part = lfirst(lc);
+
+		if (fdw_part->serverid == serverid && fdw_part->userid == userid)
+		{
+			fdw_part->modified = true;
+			found = true;
+			break;
+		}
+	}
+
+	if (!found)
+		elog(ERROR, "attempt to mark unregistered foreign server %u, user %u as modified",
+			 serverid, userid);
+}
+
+/*
+ * FdwXactShmemSize
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdw_xacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	/* Size for shared cache entry */
+	size = MAXALIGN(size);
+	size = add_size(size, hash_estimate_size(max_prepared_foreign_xacts,
+											 sizeof(FdwXactStateCacheEntry)));
+
+	return size;
+}
+
+/*
+ * FdwXactShmemInit
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of
+ * FdwXactCtlData structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdw_xacts;
+		HASHCTL		info;
+		long		max_hash_size;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->freeFdwXacts = NULL;
+		FdwXactCtl->numFdwXacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdw_xacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdw_xacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdw_xacts[cnt].status = FDW_XACT_INITIAL;
+			fdw_xacts[cnt].fxact_free_next = FdwXactCtl->freeFdwXacts;
+			FdwXactCtl->freeFdwXacts = &fdw_xacts[cnt];
+		}
+
+		/* Initialize shared state cache hash table */
+		MemSet(&info, 0, sizeof(info));
+		info.keysize = sizeof(TransactionId);
+		info.entrysize = sizeof(FdwXactStateCacheEntry);
+		max_hash_size = max_prepared_foreign_xacts;
+
+		FdwXactStateCache = ShmemInitHash("FdwXact hash",
+										  max_hash_size,
+										  max_hash_size,
+										  &info,
+										  HASH_ELEM | HASH_BLOBS);
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * PreCommit_FdwXacts
+ *
+ * This function prepares all foreign transaction participants if atomic commit
+ * is required. Otherwise commits them without preparing.
+ *
+ * If atomic commit is requested by user (that is, foreign_twophase_commit is on),
+ * every participants must enable two-phase commit. If we manage all foreign
+ * transactions involving with a transaction we can commit foreign transactions
+ * on foreign server that doesn't use two-phase commit here and commit others
+ * at post-commit phase, but we don't do that. Because (1) it doesn't satisfy
+ * the atomic commit semantics at all and (2) it requires all FDWs to register
+ * foreign server anyway, which breaks backward compatibility.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipantsForAC == NIL)
+		return;
+
+	/*
+	 * If user requires the atomic commit semantics, we don't allow COMMIT if we've
+	 * modified data on  foreign servers both that can execute two-phase commit
+	 * protocol and that cannot.
+	 */
+	if (foreign_twophase_commit == true && MyXactFlags & XACT_FLAGS_FDWNOPREPARE)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	if (ForeignTwophaseCommitRequired())
+	{
+		/* Prepare the transactions on the all foreign servers */
+		FdwXactPrepareForeignTransactions();
+	}
+	else
+	{
+		ListCell   *lc;
+
+		Assert(list_length(FdwXactParticipantsForAC) == 1);
+
+		/* Two-phase commit is not required, commit them one by one */
+		foreach(lc, FdwXactParticipantsForAC)
+		{
+			FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+			/* Commit foreign transaction */
+			if (!fdw_part->commit_foreign_xact(&fdw_part->foreign_xact))
+				ereport(ERROR,
+						(errmsg("could not commit foreign transaction on server %s",
+								fdw_part->foreign_xact.server->servername)));
+		}
+
+		/* Forget all participants */
+		FdwXactParticipantsForAC = NIL;
+	}
+}
+
+/*
+ * FdwXactPrepareForeignTransactions
+ *
+ * Prepare all foreign transaction participants.  This function creates a prepared
+ * participants chain whenever we prepared a foreign transaction. The prepared
+ * participants chain is used to access all participants of distributed transaction
+ * quickly. If any one of them fails to prepare or raises an error, we change over
+ * to aborts.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell   *lcell;
+	FdwXact		prev_fxact = NULL;
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactParticipantsForAC)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXact		fxact;
+
+		/*
+		 * Register the foreign transaction entry. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fxact = FdwXactRegisterFdwXactEntry(GetTopTransactionId(), fdw_part);
+
+		/*
+		 * Between FdwXactRegisterFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal). During abort processing,
+		 * we might try to resolve a never-prepared transaction, and get an error.
+		 * This is fine as long as the FDW provides us unique prepared transaction
+		 * identifiers.
+		 */
+		if (!fdw_part->prepare_foreign_xact(&fdw_part->foreign_xact))
+		{
+			/* Failed to prepare, change over aborts */
+			ereport(ERROR,
+					(errmsg("could not prepare transaction on foreign server %s",
+							fdw_part->foreign_xact.server->servername)));
+		}
+
+		/* Preparation is success, update its status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdw_part->fdw_xact->status = FDW_XACT_PREPARED;
+		fdw_part->fdw_xact = fxact;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Create a prepared participants chain, which is link-ed FdwXact entries
+		 * involving with this transaction. The head entry is remembered in hash
+		 * table and subsequent entries is liked from the previous entry.
+		 */
+		if (!prev_fxact)
+		{
+			FdwXactStateCacheEntry	*fxact_entry;
+			bool				found;
+
+			LWLockAcquire(FdwXactLock,LW_EXCLUSIVE);
+			fxact_entry = (FdwXactStateCacheEntry *) hash_search(FdwXactStateCache,
+																 (void *) &(fxact->local_xid),
+																 HASH_ENTER, &found);
+			LWLockRelease(FdwXactLock);
+			Assert(!found);
+
+			/* Set the first participant */
+			fxact_entry->participants = fxact;
+		}
+		else
+		{
+			/* Append others to the tail */
+			Assert(fxact->fxact_next == NULL);
+			prev_fxact->fxact_next = fxact;
+		}
+
+		prev_fxact = fxact;
+	}
+}
+
+/*
+ * FdwXactRegisterFdwXactEntry
+ *
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and will
+ * be persisted to the disk under pg_fdw_xact directory when checkpoint.
+ */
+static FdwXact
+FdwXactRegisterFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fxact;
+	FdwXactOnDiskData	*fxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact = insert_fdw_xact(MyDatabaseId, xid, fdw_part->serverid,
+							fdw_part->userid, fdw_part->fdw_xact_id);
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->registered_backend = MyBackendId;
+	fdw_part->fdw_xact = fxact;
+	LWLockRelease(FdwXactLock);
+
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdw_xact_id);
+	data_len = data_len + strlen(fdw_part->fdw_xact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fxact_file_data->dbid = MyDatabaseId;
+	fxact_file_data->local_xid = xid;
+	fxact_file_data->serverid = fdw_part->serverid;
+	fxact_file_data->userid = fdw_part->userid;
+	memcpy(fxact_file_data->fdw_xact_id, fdw_part->fdw_xact_id,
+		   strlen(fdw_part->fdw_xact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fxact_file_data, data_len);
+	fxact->insert_end_lsn = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_INSERT);
+	XLogFlush(fxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact->valid = true;
+	LWLockRelease(FdwXactLock);
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fxact_file_data);
+	return fxact;
+}
+
+/*
+ * insert_fdw_xact
+ *
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				char *fdw_xact_id)
+{
+	int i;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		fxact = FdwXactCtl->fdw_xacts[i];
+		if (fxact->dbid == dbid &&
+			fxact->local_xid == xid &&
+			fxact->serverid == serverid &&
+			fxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->freeFdwXacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fxact = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fxact->fxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->numFdwXacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts++] = fxact;
+
+	fxact->registered_backend = InvalidBackendId;
+	fxact->dbid = dbid;
+	fxact->local_xid = xid;
+	fxact->serverid = serverid;
+	fxact->userid = userid;
+	fxact->insert_start_lsn = InvalidXLogRecPtr;
+	fxact->insert_end_lsn = InvalidXLogRecPtr;
+	fxact->valid = false;
+	fxact->ondisk = false;
+	fxact->inredo = false;
+	memcpy(fxact->fdw_xact_id, fdw_xact_id, strlen(fdw_xact_id) + 1);
+
+	return fxact;
+}
+
+/*
+ * remove_fdw_xact
+ *
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdw_xact(FdwXact fdw_xact)
+{
+	int			cnt;
+
+	Assert(fdw_xact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Search the slot where this entry resided */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		if (FdwXactCtl->fdw_xacts[cnt] == fdw_xact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (cnt >= FdwXactCtl->numFdwXacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdw_xact->local_xid, fdw_xact->serverid, fdw_xact->userid)));
+
+	/* Remove the entry from active array */
+	FdwXactCtl->numFdwXacts--;
+	FdwXactCtl->fdw_xacts[cnt] = FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts];
+
+	/* Put it back into free list */
+	fdw_xact->fxact_free_next = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fdw_xact;
+
+	/* Reset informations */
+	fdw_xact->status = FDW_XACT_INITIAL;
+	fdw_xact->registered_backend = InvalidBackendId;
+	fdw_xact->fxact_next = NULL;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdw_xact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdw_xact->serverid;
+		record.dbid = fdw_xact->dbid;
+		record.xid = fdw_xact->local_xid;
+		record.userid = fdw_xact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdw_xact_remove));
+		recptr = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true if the current transaction requires foreign two-phase commit
+ * to achieve atomic commit. Foreign two-phase commit is required if we
+ * satisfy either case: we modified data on two or more foreign server, or
+ * we modified both non-temporary relation on local and data on more than
+ * one foreign server.
+ */
+bool
+ForeignTwophaseCommitRequired(void)
+{
+	int	nserverswritten = list_length(FdwXactParticipantsForAC);
+	ListCell*	lc;
+	bool		modified = false;
+
+	/* Return if not requested */
+	if (!AtomicCommitRequested())
+		return false;
+
+	/* Check if we modified data on any foreign server */
+	foreach(lc, FdwXactParticipantsForAC)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+		{
+			modified = true;
+			break;
+		}
+	}
+
+	/* We didn't modify data on any foreign server */
+	if (!modified)
+		return false;
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	return nserverswritten > 1;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * ForgetAllFdwXactParticipants
+ *
+ * Reset all the foreign transaction entries that this backend registered.
+ * If the foreign transaction has the corresponding FdwXact entry, resetting
+ * the registered_backend field means to leave that entry in unresolved state.
+ * If we leaves any entries, we update the oldest xmin of unresolved transaction
+ * so that transaction status of dangling transaction are not truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_left = 0;
+
+	if (FdwXactParticipantsForAC == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(cell, FdwXactParticipantsForAC)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+
+		/* Skip if didn't register FdwXact entry yet */
+		if (fdw_part->fdw_xact == NULL)
+			continue;
+
+		/*
+		 * There is a race condition; the entries of FdwXactParticipantsForAC
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget them. So we need to check if the entries are
+		 * still associated with the transaction.
+		 */
+		if (fdw_part->fdw_xact->registered_backend == MyBackendId)
+		{
+			fdw_part->fdw_xact->registered_backend = InvalidBackendId;
+			n_left++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Update the oldest local transaction of unresolved distributed
+	 * transaction if we leaved any FdwXact entries.
+	 */
+	if (n_left > 0)
+		FdwXactComputeRequiredXmin();
+
+	FdwXactParticipantsForAC = NIL;
+}
+
+/*
+ * AtProcExit_FdwXact
+ *
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+/*
+ * Wait for foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDW_XACT_NOT_WAITING and then change
+ * that state to FDW_XACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransactions a fdwxact resolver changes the
+ * state to FDW_XACT_WAIT_COMPLETE once foreign transactions are resolved.
+ * This backend then resets its state to FDW_XACT_NOT_WAITING.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+	ListCell	*lc;
+	List		*fdwxact_participants = NIL;
+
+	/* Quick exit if atomic commit is not requested */
+	if (!AtomicCommitRequested())
+		return;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDW_XACT_NOT_WAITING);
+
+	if (FdwXactParticipantsForAC != NIL)
+	{
+		/*
+		 * If we're waiting for foreign transactions to be resolved that
+		 * we've prepared just before, use the participants list.
+		 */
+		Assert(MyPgXact->xid == wait_xid);
+		fdwxact_participants = FdwXactParticipantsForAC;
+	}
+	else
+	{
+		FdwXactStateCacheEntry *fdwxact_entry;
+		bool found;
+
+		/*
+		 * If we're waiting for foreign transactions to be resolved that
+		 * is part of a local prepared transaction that is mark as
+		 * prepared during running, since these entries exist in the hash
+		 * table we construct the participants list from the entry.
+		 */
+		Assert(FdwXactStateCache);
+		fdwxact_entry = (FdwXactStateCacheEntry *) hash_search(FdwXactStateCache,
+															   (void *) &wait_xid,
+															   HASH_FIND, &found);
+
+		if (found)
+		{
+			FdwXact	fdwxact;
+
+			for (fdwxact = fdwxact_entry->participants;
+				 fdwxact != NULL;
+				 fdwxact = fdwxact->fxact_next)
+				fdwxact_participants = lappend(fdwxact_participants, fdwxact);
+		}
+	}
+
+	/*
+	 * Otherwise, construct the participants list by scanning the global
+	 * array. This can happen in the case where we restarts after PREPARE'd
+	 * a distributed transaction and then are trying to resolve it.
+	 */
+	if (fdwxact_participants == NIL)
+		fdwxact_participants = get_fdw_xacts(MyDatabaseId, wait_xid,
+											 InvalidOid, InvalidOid, true);
+
+	/* Exit if we found no foreign transaction to resolve */
+	if (fdwxact_participants == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(lc, fdwxact_participants)
+	{
+		FdwXact fdw_xact = (FdwXact) lfirst(lc);
+
+		/* Don't overwrite status if fate has been determined */
+		if (fdw_xact->status == FDW_XACT_PREPARED)
+			fdw_xact->status = (is_commit ?
+								FDW_XACT_COMMITTING_PREPARED :
+								FDW_XACT_ABORTING_PREPARED);
+	}
+
+	/* Set backend status and enqueue itself */
+	MyProc->fdwXactState = FDW_XACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	FdwXactQueueInsert();
+	LWLockRelease(FdwXactLock);
+
+	/* Launch a resolver process if not yet, or wake it up */
+	fdwxact_maybe_launch_resolver(false);
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDW_XACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDW_XACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDW_XACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+
+	/*
+	 * Forget the list of locked entries, also means that the entries
+	 * that could not resolved are remained as dangling transactions.
+	 */
+	ForgetAllFdwXactParticipants();
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Acquire FdwXactLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Insert MyProc into the tail of FdwXactQueue.
+ */
+static void
+FdwXactQueueInsert(void)
+{
+	SHMQueueInsertBefore(&(FdwXactRslvCtl->FdwXactQueue),
+						 &(MyProc->fdwXactLinks));
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * Create and initialize an FdwXactResolveState which is used
+ * for resolution of foreign transactions.
+ */
+FdwXactResolveState *
+CreateFdwXactResolveState(void)
+{
+	FdwXactResolveState *frstate = palloc0(sizeof(FdwXactResolveState));
+
+	frstate->dbid = MyDatabaseId;
+	frstate->fdwxact = NULL;
+	frstate->waiter = NULL;
+
+	return frstate;
+}
+
+/*
+ * Resolve one distributed transaction. The target distributed transaction
+ * is fetched from shmem queue and its participants are fetched from either
+ * shmem hash table or global array. Release the waiter and return true only
+ * if we resolved the all of the foreign transaction participants. Return
+ * false if we flied to resolve any of them.
+ *
+ * To ensure the order of registered distributed transaction to the queue, we
+ * must not go the next distributed transaction until all of participants are
+ * resolved. The failed foreign transactions will be retried at the next execution.
+ */
+bool
+FdwXactResolveDistributedTransaction(FdwXactResolveState *frstate)
+{
+	FdwXactStateCacheEntry	*fdwxact_entry = NULL;
+	volatile FdwXact	fdwxacts_failed_to_resolve = NULL;
+	bool				all_resolved = false;
+
+	Assert(frstate->dbid == MyDatabaseId);
+
+	/* Get a new waiter, if not exists */
+	if (frstate->waiter == NULL)
+	{
+		PGPROC	*proc;
+
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+		/* Fetch a waiter from beginning of the queue */
+		while ((proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->FdwXactQueue),
+											   &(FdwXactRslvCtl->FdwXactQueue),
+											   offsetof(PGPROC, fdwXactLinks))) != NULL)
+		{
+			/* Found a waiter */
+			if (proc->databaseId == frstate->dbid)
+				break;
+		}
+
+		LWLockRelease(FdwXactLock);
+
+		/* If no waiter, there is no job */
+		if (!proc)
+			return false;
+
+		Assert(TransactionIdIsValid(proc->fdwXactWaitXid));
+		frstate->waiter = proc;
+	}
+
+	/* Get foreign transaction participants */
+	if (frstate->fdwxact == NULL)
+	{
+		bool found;
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+		/* Search FdwXact entries from the hash table by the local transaction id */
+		fdwxact_entry =
+			(FdwXactStateCacheEntry *) hash_search(FdwXactStateCache,
+												   (void *) &(frstate->waiter->fdwXactWaitXid),
+												   HASH_FIND, &found);
+
+		if (found)
+			frstate->fdwxact = fdwxact_entry->participants;
+		else
+		{
+			int i;
+			FdwXact entries_to_resolve = NULL;
+			FdwXact prev_fx = NULL;
+
+			/*
+			 * The fdwxact entry doesn't exist in the hash table in case where
+			 * a prepared transaction is resolved after recovery. In this case,
+			 * we construct a list of fdw xact entries by scanning over the
+			 * FdwXactCtl->fdw_xacts list.
+			 */
+			for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+			{
+				FdwXact fdw_xact = FdwXactCtl->fdw_xacts[i];
+
+				if (fdw_xact->dbid == frstate->dbid &&
+					fdw_xact->local_xid == frstate->waiter->fdwXactWaitXid)
+				{
+					if (!entries_to_resolve)
+						entries_to_resolve = fdw_xact;
+
+					/* Link from previous entry to this entry */
+					if (prev_fx)
+						prev_fx->fxact_next = fdw_xact;
+
+					prev_fx = fdw_xact;
+				}
+			}
+
+			frstate->fdwxact = entries_to_resolve;
+		}
+
+		LWLockRelease(FdwXactLock);
+	}
+
+	Assert(frstate->fdwxact != NULL);
+
+	/* Resolve all foreign transactions one by one */
+	while (frstate->fdwxact != NULL)
+	{
+		volatile FdwXact cur_fdwxact = frstate->fdwxact;
+		volatile FdwXact fdwxact_next = NULL;
+
+		/*
+		 * Remember the next FdwXact entry to resolve as the current entry will
+		 * be removed after resolved from the list.
+		 */
+		fdwxact_next = cur_fdwxact->fxact_next;
+
+		/* Resolve a foreign transaction */
+		if (!FdwXactResolveForeignTransaction(cur_fdwxact))
+		{
+			ForeignServer *fserver;
+
+			CHECK_FOR_INTERRUPTS();
+
+			/* Failed to resolve. Remember it for the next execution */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			if (fdwxacts_failed_to_resolve == NULL)
+			{
+				/*
+				 * For the first failed entry, reset its next pointer
+				 * and append it to the head of list.
+				 */
+				cur_fdwxact->fxact_next = NULL;
+				fdwxacts_failed_to_resolve = cur_fdwxact;
+			}
+			else
+			{
+				FdwXact fx = fdwxacts_failed_to_resolve;
+
+				/* Append the entry at the tail */
+				while (fx->fxact_next != NULL)
+					fx = fx->fxact_next;
+				fx->fxact_next = cur_fdwxact;
+			}
+			LWLockRelease(FdwXactLock);
+
+			fserver = GetForeignServer(cur_fdwxact->serverid);
+			ereport(LOG,
+					(errmsg("could not resolve a foreign transaction on server \"%s\"",
+							fserver->servername),
+					 errdetail("local transaction id is %u, connected by user id %u",
+							   cur_fdwxact->local_xid, cur_fdwxact->userid)));
+		}
+		else
+		{
+			/* Resolved. Update the cache entry if it's valid */
+			if (fdwxact_entry)
+				fdwxact_entry->participants = fdwxact_next;
+
+			elog(DEBUG2, "resolved a foreign transaction xid %u, serverid %d, userid %d",
+				 cur_fdwxact->local_xid, cur_fdwxact->serverid, cur_fdwxact->userid);
+		}
+
+		/* Advance the resolution status to the next */
+		frstate->fdwxact = fdwxact_next;
+	}
+
+	all_resolved = (fdwxacts_failed_to_resolve == NULL);
+
+	if (all_resolved)
+	{
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+		/* Remove the state cache entry from shmem hash table */
+		hash_search(FdwXactStateCache, (void *) &(frstate->waiter->fdwXactWaitXid),
+					HASH_REMOVE, NULL);
+
+		/*
+		 * Remove waiter from shmem queue, if not detached yet. The waiter
+		 * could already be detached if user cancelled to wait before
+		 * resolution.
+		 */
+		if (!SHMQueueIsDetached(&(frstate->waiter->fdwXactLinks)))
+		{
+			TransactionId	wait_xid = frstate->waiter->fdwXactWaitXid;
+
+			SHMQueueDelete(&(frstate->waiter->fdwXactLinks));
+
+			pg_write_barrier();
+
+			/* Set state to complete */
+			frstate->waiter->fdwXactState = FDW_XACT_WAIT_COMPLETE;
+
+			/* Wake up the waiter only when we have set state and removed from queue */
+			SetLatch(&(frstate->waiter->procLatch));
+
+			elog(DEBUG2, "released a proc xid %u", wait_xid);
+		}
+
+		LWLockRelease(FdwXactLock);
+
+		/* Reset resolution state */
+		frstate->waiter = NULL;
+		Assert(frstate->fdwxact == NULL);
+	}
+	else
+	{
+		/*
+		 * Update the fdwxact entry we're processing so that the failed
+		 * fdwxact entries will be processed again.
+		 */
+		frstate->fdwxact = fdwxacts_failed_to_resolve;
+	}
+
+	return all_resolved;
+}
+
+/*
+ * Resolve all dangling foreign transactions on the given database. Get
+ * all dangling foreign transactions from shmem global array and resolve
+ * them one by one.
+ *
+ * Unlike FdwXactResolveDistributedTransaction, for dangling transaction
+ * resolution, we don't bother the order of resolution because these entries
+ * already got out of order. So if failed to resolve a foreign transaction,
+ * we can go to the next foreign transaction that might associates with
+ * an another distributed transaction.
+ */
+void
+FdwXactResolveAllDanglingTransactions(Oid dbid)
+{
+	List		*dangling_fdwxacts = NIL;
+	ListCell	*cell;
+	bool		n_resolved = 0;
+	int			i;
+
+	Assert(OidIsValid(dbid));
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/*
+	 * Walk over the global array to make the list of dangling transactions
+	 * of which corresponding local transaction is on the given database.
+	 */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fxact = FdwXactCtl->fdw_xacts[i];
+
+		/*
+		 * Append the fdwxact entry on the given database to the list if
+		 * it's handled by nobody and the corresponding local transaction
+		 * is not part of the prepared transaction.
+		 */
+		if (fxact->dbid == dbid &&
+			fxact->registered_backend == InvalidBackendId &&
+			!TwoPhaseExists(fxact->local_xid))
+			dangling_fdwxacts = lappend(dangling_fdwxacts, fxact);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/* Return if there is no foreign transaction we need to resolve */
+	if (dangling_fdwxacts == NIL)
+		return;
+
+	foreach(cell, dangling_fdwxacts)
+	{
+		FdwXact fdwxact = (FdwXact) lfirst(cell);
+
+		if (!FdwXactResolveForeignTransaction(fdwxact))
+		{
+			ForeignServer *fserver = GetForeignServer(fdwxact->serverid);
+
+			/*
+			 * If failed to resolve this foreign transaction we skip it in
+			 * this resolution cycle. Try to resolve again in next cycle.
+			 */
+			ereport(LOG,
+					(errmsg("could not resolve a dangling foreign transaction on server \"%s\"",
+							fserver->servername),
+					 errdetail("local transaction id is %u, connected by user id %u",
+							   fdwxact->local_xid, fdwxact->userid)));
+			continue;
+		}
+
+		n_resolved++;
+	}
+
+	list_free(dangling_fdwxacts);
+
+	elog(DEBUG2, "resolved %d dangling foreign xacts", n_resolved);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ *
+ * In commit case, we have already prepared transactions on the foreign
+ * servers during pre-commit. And that prepared transactions will be
+ * resolved by the resolver process. So we don't do anything about the
+ * foreign transaction.
+ *
+ * In abort case, user requested rollback or we changed over rollback
+ * due to error during commit. To close current foreign transaction anyway
+ * we call rollback API to every foreign transaction. If we raised an error
+ * during preparing and came to here, it's possible that some entries of
+ * FdwXactParticipants already registered its FdwXact entry. If there is
+ * we leave them as dangling transaction and ask the resolver process to
+ * process them.
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		int left_fdwxacts = 0;
+
+		foreach (lcell, FdwXactParticipantsForAC)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * Count FdwXact entries that we registered to shared memory array
+			 * in this transaction.
+			 */
+			if (fdw_part->fdw_xact)
+			{
+				/*
+				 * The status of foreign transaction must be either preparing
+				 * or prepared. In any case, since we have registered FdwXact
+				 * entry we leave them to the resolver process. For the preparing
+				 * state, since the foreign transaction might not close yet we
+				 * fall through and call rollback API. For the prepared state,
+				 * since the foreign transaction has closed we don't need to do
+				 * anything.
+				 */
+				Assert(fdw_part->fdw_xact->status == FDW_XACT_PREPARING ||
+					   fdw_part->fdw_xact->status == FDW_XACT_PREPARED);
+
+				left_fdwxacts++;
+				if (fdw_part->fdw_xact->status == FDW_XACT_PREPARED)
+					continue;
+			}
+
+			/*
+			 * Rollback all current foreign transaction. Since we're rollbacking
+			 * the transaction it's too late even if we raise an error here.
+			 * So we log it as warning.
+			 */
+			if (!fdw_part->rollback_foreign_xact(&fdw_part->foreign_xact))
+				ereport(WARNING,
+						(errmsg("could not abort transaction on server \"%s\"",
+								fdw_part->foreign_xact.server->servername)));
+		}
+
+		/* If we left some FdwXact entries, ask the resolver process */
+		if (left_fdwxacts > 0)
+		{
+			ereport(WARNING,
+					(errmsg("left %u foreign transactions in in-doubt status",
+							left_fdwxacts)));
+			fdwxact_maybe_launch_resolver(true);
+		}
+	}
+
+	ForgetAllFdwXactParticipants();
+}
+
+/*
+ * AtPrepare_FdwXacts
+ *
+ * If there are foreign servers involved in the transaction, this function
+ * prepares transactions on those servers.
+ *
+ * Note that it can happen that the transaction aborts after we prepared part
+ * of participants. In this case since we can change to abort we cannot forget
+ * FdwXactParticipantsForAC here. These are processed by the resolver process
+ * during aborting, or at EOXact_FdwXacts.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipantsForAC == NIL)
+		return;
+
+	/*
+	 * We cannot prepare distributed transaction if any foreign server of
+	 * participants in the transaction isn't capable of two-phase commit.
+	 */
+	if ((MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_T_R_INTEGRITY_CONSTRAINT_VIOLATION),
+				 errmsg("can not prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * FdwXactResolveForeignTransaction
+ *
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * handler routine. The foreign transaction can be a dangling transaction
+ * that is not interested by nobody. If the fate of foreign transaction is
+ * not determined yet, it'sdetermined according to the status of corresponding
+ * local transaction.
+ *
+ * If the resolution is successful, remove the foreign transaction entry from
+ * the shared memory and also remove the corresponding on-disk file.
+ */
+static bool
+FdwXactResolveForeignTransaction(FdwXact fdwxact)
+{
+	bool		resolved;
+	bool		is_commit;
+	ForeignServer		*fserver;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	FdwRoutine			*fdw_routine;
+	ForeignTransaction	foreign_xact;
+
+	Assert(fdwxact);
+
+	/*
+	 * Determine whether we commit or abort this foreign transaction.
+	 */
+	if (fdwxact->status == FDW_XACT_COMMITTING_PREPARED)
+		is_commit = true;
+	else if (fdwxact->status == FDW_XACT_ABORTING_PREPARED)
+		is_commit = false;
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	else if (TransactionIdDidCommit(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_COMMITTING_PREPARED;
+		is_commit = true;
+	}
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_ABORTING_PREPARED;
+		is_commit = false;
+	}
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+		is_commit = false;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction
+	 * state is neither committing or aborting. This should not
+	 * happen because we cannot determine to do commit or abort for
+	 * foreign transaction associated with the in-progress local
+	 * transaction.
+	 */
+	else
+		ereport(ERROR,
+				(errmsg("cannot resolve foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid)));
+
+	/* Construct foreign server connection information for passing to API */
+	fserver = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(fserver->fdwid);
+	user_mapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+	foreign_xact.server = fserver;
+	foreign_xact.usermapping = user_mapping;
+	foreign_xact.fx_id = fdwxact->fdw_xact_id;
+
+	/* Resolve the foreign transaction */
+	Assert(fdw_routine->ResolveForeignTransaction);
+	resolved = fdw_routine->ResolveForeignTransaction(&foreign_xact,
+													  is_commit);
+
+	if (!resolved)
+	{
+		ForeignServer *fserver = GetForeignServer(fdwxact->serverid);
+		ereport(ERROR,
+				(errmsg("could not %s a prepared foreign transaction on server \"%s\"",
+						is_commit ? "commit" : "rollback", fserver->servername),
+				 errdetail("local transaction id is %u, connected by user id %u",
+						   fdwxact->local_xid, fdwxact->userid)));
+	}
+	else
+	{
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		if (fdwxact->ondisk)
+			RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+							  fdwxact->serverid, fdwxact->userid,
+							  true);
+		remove_fdw_xact(fdwxact);
+		LWLockRelease(FdwXactLock);
+	}
+
+	return resolved;
+}
+
+/*
+ * Return one FdwXact entry that matches to given arguments, otherwise
+ * return NULL. Since this function search FdwXact entry by unique key
+ * all arguments should be valid.
+ */
+static FdwXact
+get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				 bool need_lock)
+{
+	List	*fdw_xact_list;
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, need_lock);
+
+	/* Could not find entry */
+	if (fdw_xact_list == NIL)
+		return NULL;
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdw_xact_list) == 1);
+
+	return (FdwXact) linitial(fdw_xact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdw_xact_exists(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdw_xact_list;
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, true);
+
+	return fdw_xact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_prepared_fdw_xacts.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdw_xacts(int *length)
+{
+	List		*all_fdw_xacts;
+	ListCell	*lc;
+	FdwXact		fdw_xacts;
+	int			num_fdw_xacts = 0;
+
+	Assert(length != NULL);
+
+	/* Get all entries */
+	all_fdw_xacts = get_fdw_xacts(InvalidOid, InvalidTransactionId,
+								  InvalidOid, InvalidOid, true);
+
+	if (all_fdw_xacts == NIL)
+	{
+		*length = 0;
+		return NULL;
+	}
+
+	fdw_xacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdw_xacts));
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdw_xacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdw_xacts + num_fdw_xacts, fx,
+			   sizeof(FdwXactData));
+		num_fdw_xacts++;
+	}
+
+	*length = num_fdw_xacts;
+	list_free(all_fdw_xacts);
+
+	return fdw_xacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return
+ * NIL.
+ */
+static List*
+get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			  bool need_lock)
+{
+	int i;
+	List	*fdw_xact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact	fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool	matches = true;
+
+		/* xid */
+		if (xid != InvalidTransactionId && xid != fdw_xact->local_xid)
+			matches = false;
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdw_xact->dbid != dbid)
+			matches = false;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdw_xact->serverid)
+			matches = false;
+
+		/* userid */
+		if (OidIsValid(userid) && fdw_xact->userid != userid)
+			matches = false;
+
+		/* Append it if matched */
+		if (matches)
+			fdw_xact_list = lappend(fdw_xact_list, fdw_xact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdw_xact_list;
+}
+
+/*
+ * fdw_xact_redo
+ * Apply the redo log for a foreign transaction.
+ */
+void
+fdw_xact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDW_XACT_REMOVE)
+	{
+		xl_fdw_xact_remove *record = (xl_fdw_xact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. Returned string
+ * value is used to identify foreign transaction. The identifier should not
+ * be same as any other concurrent prepared transaction identifier.
+ *
+ * To make the foreign transactionid, we should ideally use something like
+ * UUID, which gives unique ids with high probability, but that may be expensive
+ * here and UUID extension which provides the function to generate UUID is
+ * not part of the core code.
+ */
+static char *
+generate_fdw_xact_identifier(Oid serverid, Oid userid)
+{
+	char*	fdw_xact_id;
+
+	fdw_xact_id = (char *)palloc(FDW_XACT_ID_MAX_LEN * sizeof(char));
+
+	snprintf(fdw_xact_id, FDW_XACT_ID_MAX_LEN, "%s_%ld_%d_%d",
+			 "fx", Abs(random()), serverid, userid);
+	fdw_xact_id[strlen(fdw_xact_id)] = '\0';
+
+	return fdw_xact_id;
+}
+
+/*
+ * CheckPointFdwXact
+ *
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * In order to avoid disk I/O while holding a light weight lock, the function
+ * first collects the files which need to be synced under FdwXactLock and then
+ * syncs them after releasing the lock. This approach creates a race condition:
+ * after releasing the lock, and before syncing a file, the corresponding
+ * foreign transaction entry and hence the file might get removed. The function
+ * checks whether that's true and ignores the error if so.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdw_xacts = 0;
+
+	/* Quick get-away, before taking lock */
+	if (max_prepared_foreign_xacts <= 0)
+		return;
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/* Another quick, before we allocate memory */
+	if (FdwXactCtl->numFdwXacts <= 0)
+	{
+		LWLockRelease(FdwXactLock);
+		return;
+	}
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		FdwXact		fxact = FdwXactCtl->fdw_xacts[cnt];
+
+		if ((fxact->valid || fxact->inredo) &&
+			!fxact->ondisk &&
+			fxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fxact->dbid, fxact->local_xid,
+								fxact->serverid, fxact->userid,
+								buf, len);
+			fxact->ondisk = true;
+			fxact->insert_start_lsn = InvalidXLogRecPtr;
+			fxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdw_xacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDW_XACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdw_xacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdw_xacts,
+							 serialized_fdw_xacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDW_XACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDW_XACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * ProcessFdwXactBuffer
+ *
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid = ShmemVariableCache->nextXid;
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(insert_start_lsn != InvalidXLogRecPtr);
+
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid, true);
+		if (buf == NULL)
+		{
+			ereport(WARNING,
+					(errmsg("removing corrupt fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+			return NULL;
+		}
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return thecontents in
+ * a structure allocated in-memory. Otherwise return NULL. The structure can
+ * be later freed by the caller.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				bool give_warnings)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+	{
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not stat FDW transaction state file \"%s\": %m",
+							path)));
+		return NULL;
+	}
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdw_xact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+	{
+		CloseTransientFile(fd);
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+		return NULL;
+	}
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+	{
+		CloseTransientFile(fd);
+		return NULL;
+	}
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_READ);
+	if (read(fd, buf, stat.st_size) != stat.st_size)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not read FDW transaction state file \"%s\": %m",
+					  path)));
+		return NULL;
+	}
+
+	pgstat_report_wait_end();
+	CloseTransientFile(fd);
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+	{
+		pfree(buf);
+		return NULL;
+	}
+
+	/* Check if the contents is an expected data */
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fxact_file_data->dbid  != dbid ||
+		fxact_file_data->serverid != serverid ||
+		fxact_file_data->userid != userid ||
+		fxact_file_data->local_xid != xid)
+	{
+		ereport(WARNING,
+			(errmsg("invalid foreign transaction state file \"%s\"",
+					path)));
+		CloseTransientFile(fd);
+		pfree(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+/*
+ * PrescanFdwXacts
+ *
+ * Scan the all foreign transactions directory for oldest active transaction.
+ * This is run during database startup, after we completed reading WAL.
+ * ShmemVariableCache->nextXid has been set to one more than the highest XID
+ * for which evidence exists in WAL.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	TransactionId nextXid = ShmemVariableCache->nextXid;
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+		 strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			TransactionId local_xid;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/*
+			 * Remove a foreign prepared transaction file corresponding to an
+			 * XID, which is too new.
+			 */
+			if (TransactionIdFollowsOrEquals(local_xid, nextXid))
+			{
+				ereport(WARNING,
+						(errmsg("removing future foreign prepared transaction file \"%s\"",
+								clde->d_name)));
+				RemoveFdwXactFile(dbid, local_xid, serverid, userid, true);
+				continue;
+			}
+
+			if (TransactionIdPrecedesOrEquals(local_xid, oldestActiveXid))
+				oldestActiveXid = local_xid;
+		}
+	}
+
+	FreeDir(cldir);
+	return oldestActiveXid;
+}
+
+/*
+ * restoreFdwXactData
+ *
+ * Scan pg_fdw_xact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid, bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * FdwXactRedoAdd
+ *
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fxact = insert_fdw_xact(fxact_data->dbid, fxact_data->local_xid,
+							fxact_data->serverid, fxact_data->userid,
+							fxact_data->fdw_xact_id);
+
+	/*
+	 * Set status as preparing, since we do not know the xact status
+	 * right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->insert_start_lsn = start_lsn;
+	fxact->insert_end_lsn = end_lsn;
+	fxact->inredo = true;	/* added in redo */
+	fxact->valid = false;
+	fxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * FdwXactRedoRemove
+ *
+ * Remove the corresponding fdw_xact entry from FdwXactCtl.
+ * Also remove fdw_xact file if a foreign transaction was saved
+ * via an earlier checkpoint.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdw_xact(dbid, xid, serverid, userid,
+							   false);
+
+	if (fdwxact == NULL)
+		return;
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdw_xact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(bool *newval, void **extra, GucSource source)
+{
+	/* Parameter check */
+	if (*newval &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_xacts\" or \"max_foreign_xact_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdw_xacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_prepared_fdw_xacts(PG_FUNCTION_ARGS)
+{
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdw_xacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdw_xacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(6, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdw_xacts = get_all_fdw_xacts(&num_fdw_xacts);
+		status->num_xacts = num_fdw_xacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdw_xact = &status->fdw_xacts[status->cur_xact++];
+		Datum		values[6];
+		bool		nulls[6];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdw_xact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdw_xact->dbid);
+		values[1] = TransactionIdGetDatum(fdw_xact->local_xid);
+		values[2] = ObjectIdGetDatum(fdw_xact->serverid);
+		values[3] = ObjectIdGetDatum(fdw_xact->userid);
+		switch (fdw_xact->status)
+		{
+			case FDW_XACT_PREPARING:
+				xact_status = "prepared";
+				break;
+			case FDW_XACT_COMMITTING_PREPARED:
+				xact_status = "committing";
+				break;
+			case FDW_XACT_ABORTING_PREPARED:
+				xact_status = "aborting";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		/* should this be really interpreted by FDW */
+		values[5] = PointerGetDatum(cstring_to_text_with_len(fdw_xact->fdw_xact_id,
+															 strlen(fdw_xact->fdw_xact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+	bool			ret;
+
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, true);
+	if (fdwxact == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("could not find foreign transaction entry"))));
+
+	ret = FdwXactResolveForeignTransaction(fdwxact);
+
+	PG_RETURN_BOOL(ret);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, false);
+	if (fdwxact == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("could not find foreign transaction entry"))));
+
+	remove_fdw_xact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_VOID();
+}
diff --git a/src/backend/foreign/fdwxact_launcher.c b/src/backend/foreign/fdwxact_launcher.c
new file mode 100644
index 0000000..6782c33
--- /dev/null
+++ b/src/backend/foreign/fdwxact_launcher.c
@@ -0,0 +1,587 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * There is a shared memory area where the information of resolver process
+ * is stored. Requesting of starting new resolver process by backend process
+ * is done via that shared memory area. Note that the launcher is assuming
+ * that there is no more than one starting request for a database.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/foreign/fdwxact_launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "foreign/fdwxact.h"
+#include "foreign/fdwxact_launcher.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/resolver_internal.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid, int slot);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+Datum pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS);
+
+/*
+ * Wake up the launcher process.
+ */
+void
+FdwXactLauncherWakeup(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR1);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->FdwXactQueue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+
+		now = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			bool launched;
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+				last_start_time = now;
+		}
+		else
+		{
+			/*
+			 * The wint in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver worker
+ * if not running yet. A foreign transaction resolver worker is responsible
+ * for resolution of foreign transaction that are registered on a database.
+ * So if a resolver worker already is launched, we don't need to launch new
+ * one.
+ */
+void
+fdwxact_maybe_launch_resolver(bool ignore_error)
+{
+	FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->pid != InvalidPid &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/*
+	 * If we found the resolver for my database, we don't need to launch new
+	 * one but wake running worker up.
+	 */
+	if (found)
+	{
+		SetLatch(resolver->latch);
+
+		elog(DEBUG1, "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		return;
+	}
+
+	/* Looking for unused worker slot */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	/*
+	 * However if there are no more free worker slots, inform user about it before
+	 * exiting.
+	 */
+	if (!found)
+	{
+		LWLockRelease(FdwXactResolverLock);
+
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+		return;
+	}
+
+	Assert(resolver->pid == InvalidPid);
+
+	/* Found a new resolver process */
+	resolver->dbid = MyDatabaseId;
+	resolver->in_use = true;
+
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Wake up launcher */
+	FdwXactLauncherWakeup();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid' at 'slot' if given. If slot is negative value we find an unused slot.
+ * Note that caller must hold FdwXactResolverLock in exclusive mode.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid, int slot)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int launch_slot = slot;
+
+	/* If slot number is invalid, we find an unused slot */
+	if (launch_slot < 0)
+	{
+		int i;
+
+		for (i = 0; i < max_foreign_xact_resolvers; i++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+			if (resolver->in_use && resolver->dbid == dbid)
+				return;
+
+			if (!resolver->in_use)
+			{
+				launch_slot = i;
+				break;
+			}
+		}
+	}
+
+	/* No unused found */
+	if (launch_slot < 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[launch_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_main_arg = Int32GetDatum(launch_slot);
+	bgw.bgw_notify_pid = (Datum) 0;
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch all foreign transaction resolvers that are required by backend process
+ * but not running.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	int i, j;
+	int num_launches = 0;
+	int num_unused_slots = 0;
+	int num_dbs = 0;
+	bool launched = false;
+	Oid	*dbs_to_launch;
+	Oid *dbs_having_worker = palloc0(sizeof(Oid) * max_foreign_xact_resolvers);
+
+	/*
+	 * Launch resolver workers on the databases that are requested
+	 * by backend processes.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* Remember unused worker slots */
+		if (!resolver->in_use)
+			num_unused_slots++;
+
+		/* Remember databases that are having a resolve worker */
+		if (OidIsValid(resolver->dbid))
+			dbs_having_worker[num_dbs++] = resolver->dbid;
+
+		/* Launch new foreign transaction resolver worker on the database */
+		if (resolver->in_use &&
+			OidIsValid(resolver->dbid) &&
+			resolver->pid == InvalidPid)
+		{
+			fdwxact_launch_resolver(resolver->dbid, i);
+			launched = true;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* There is no unused slot, exit */
+	if (num_unused_slots == 0)
+		return launched;
+
+	dbs_to_launch = (Oid *) palloc(sizeof(Oid) * num_unused_slots);
+
+	/*
+	 * If there is unused slot, we can launch foreign transaction resolver
+	 * on databases that has unresolved foreign transaction but doesn't
+	 * have any resolver. This usually happens when resolvers crash for
+	 * whatever reason. Scanning all FdwXact entries could takes time but
+	 * since this is a relaunch case it's not harmless.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool found = false;
+
+		if (num_launches > num_unused_slots)
+			break;
+
+		for (j = 0; j < num_dbs; j++)
+		{
+			if (dbs_having_worker[j] == fdw_xact->dbid)
+			{
+				found = true;
+				break;
+			}
+		}
+
+		if (found)
+			continue;
+
+		dbs_to_launch[num_launches++] = fdw_xact->dbid;
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* Launch resolver process for a database at any worker slot */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < num_launches; i++)
+	{
+		fdwxact_launch_resolver(dbs_to_launch[i], -1);
+		launched = true;
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+
+/*
+ * Returns activity of foreign transaction resolvers, including pids, the number
+ * of tasks and the last resolution time.
+ */
+Datum
+pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		if (resolver->pid == 0)
+		{
+			SpinLockRelease(&(MyFdwXactResolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/foreign/fdwxact_resolver.c b/src/backend/foreign/fdwxact_resolver.c
new file mode 100644
index 0000000..7f7ff8f
--- /dev/null
+++ b/src/backend/foreign/fdwxact_resolver.c
@@ -0,0 +1,310 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for every databases.
+ *
+ * A resolver process continues to resolve foreign transactions on a database
+ * It resolves two types of foreign transactions: on-line foreign transaction
+ * and dangling foreign transaction. The on-line foreign transaction is a
+ * foreign transaction that a concurrent backend process is waiting for
+ * resolution. The dangling transaction is a foreign transaction that corresponding
+ * distributed transaction ended up in in-doubt state. A resolver process
+ * doesn' exit as long as there is at least one unresolved foreign transaction
+ * on the database even if the timeout has come.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/foreign/fdwxact_resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "foreign/fdwxact.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/fdwxact_launcher.h"
+#include "foreign/resolver_internal.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+
+//static MemoryContext ResolveContext = NULL;
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FdwXactRslvLoop(void);
+static long FdwXactRslvComputeSleepTime(TimestampTz now);
+static void FdwXactRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+	FdwXactLauncherWakeup();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	TIMESTAMP_NOBEGIN(MyFdwXactResolver->last_resolved_time);
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FdwXactRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FdwXactRslvLoop(void)
+{
+	FdwXactResolveState *fstate;
+
+	/* Create an FdwXactResolveState */
+	fstate = CreateFdwXactResolveState();
+
+	/* Enter main loop */
+	for (;;)
+	{
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time;
+		bool		resolved;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		/* Resolve a distributed transaction */
+		StartTransactionCommand();
+		resolved = FdwXactResolveDistributedTransaction(fstate);
+		CommitTransactionCommand();
+
+		now = GetCurrentTimestamp();
+
+		/* Update my state */
+		if (resolved)
+			MyFdwXactResolver->last_resolved_time = now;
+
+		/* Check for fdwxact resolver timeout */
+		FdwXactRslvCheckTimeout(now);
+
+		/*
+		 * If we have resolved any distributed transaction we go the next
+		 * without both resolving dangling transaction and sleeping because
+		 * there might be other on-line transactions waiting to be resolved.
+		 */
+		if (!resolved)
+		{
+			/* Resolve dangling transactions as mush as possible */
+			StartTransactionCommand();
+			FdwXactResolveAllDanglingTransactions(MyDatabaseId);
+			CommitTransactionCommand();
+
+			sleep_time = FdwXactRslvComputeSleepTime(now);
+
+			rc = WaitLatch(MyLatch,
+						   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+						   sleep_time,
+						   WAIT_EVENT_FDW_XACT_RESOLVER_MAIN);
+
+			if (rc & WL_POSTMASTER_DEATH)
+				proc_exit(1);
+		}
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FdwXactRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	/*
+	 * Reached to the timeout. We exit if there is no more both pending on-line
+	 * transactions and dangling transactions.
+	 */
+	if (!fdw_xact_exists(InvalidTransactionId, MyDatabaseId, InvalidOid,
+						 InvalidOid))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyFdwXactResolver->dbid))));
+		CommitTransactionCommand();
+
+		fdwxact_resolver_detach();
+		proc_exit(0);
+	}
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. Return the sleep time
+ * in milliseconds, -1 means that we reached to the timeout and should exits
+ */
+static long
+FdwXactRslvComputeSleepTime(TimestampTz now)
+{
+	static TimestampTz	wakeuptime = 0;
+	long	sleeptime;
+	long	sec_to_timeout;
+	int		microsec_to_timeout;
+
+	if (now >= wakeuptime)
+		wakeuptime = TimestampTzPlusMilliseconds(now,
+												 foreign_xact_resolution_retry_interval);
+
+	/* Compute relative time until wakeup. */
+	TimestampDifference(now, wakeuptime,
+						&sec_to_timeout, &microsec_to_timeout);
+
+	sleeptime = sec_to_timeout * 1000 + microsec_to_timeout / 1000;
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index eac78a5..1873a24 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -155,6 +155,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMapping - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index f651bb4..cfd73f5 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -16,6 +16,8 @@
 
 #include "libpq/pqsignal.h"
 #include "access/parallel.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/fdwxact_launcher.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "port/atomics.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index bbe7361..757d060 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3492,6 +3492,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_LAUNCHER_MAIN:
 			event_name = "LogicalLauncherMain";
 			break;
@@ -3683,6 +3689,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -3898,6 +3907,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDW_XACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index a4b53b3..1c9ca53 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -99,6 +99,8 @@
 #include "catalog/pg_control.h"
 #include "common/file_perm.h"
 #include "common/ip.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/fdwxact_launcher.h"
 #include "lib/ilist.h"
 #include "libpq/auth.h"
 #include "libpq/libpq.h"
@@ -905,6 +907,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires maX_foreign_xact_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -980,12 +986,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index 59c003d..ce09a2a 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -154,6 +154,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDW_XACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 0c86a58..5f321fe 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -22,6 +22,7 @@
 #include "access/subtrans.h"
 #include "access/twophase.h"
 #include "commands/async.h"
+#include "foreign/fdwxact_launcher.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "postmaster/autovacuum.h"
@@ -150,6 +151,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
 		size = add_size(size, BackendRandomShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -270,6 +273,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 	SyncScanShmemInit();
 	AsyncShmemInit();
 	BackendRandomShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index bd20497..d05d89f 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -90,6 +90,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -245,6 +247,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1318,6 +1321,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	volatile TransactionId replication_slot_xmin = InvalidTransactionId;
 	volatile TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	volatile TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1379,6 +1383,7 @@ GetOldestXmin(Relation rel, int flags)
 	/* fetch into volatile var while ProcArrayLock is held */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1429,6 +1434,15 @@ GetOldestXmin(Relation rel, int flags)
 		result = replication_slot_xmin;
 
 	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDW_XACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
+	/*
 	 * After locks have been released and defer_cleanup_age has been applied,
 	 * check whether we need to back up further to make logical decoding
 	 * possible. We need to do so if we're computing the global limit (rel =
@@ -3005,6 +3019,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index e6025ec..a42d06e 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -50,3 +50,5 @@ OldSnapshotTimeMapLock				42
 BackendRandomLock					43
 LogicalRepWorkerLock				44
 CLogTruncationLock					45
+FdwXactLock					46
+FdwXactResolverLock			47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 6f30e08..577d2ff 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -38,6 +38,7 @@
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
+#include "foreign/fdwxact.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "postmaster/autovacuum.h"
@@ -397,6 +398,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* initialize fields for fdw xact */
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -797,6 +802,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index f413395..2b3dee5 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -43,6 +43,8 @@
 #include "commands/async.h"
 #include "commands/prepare.h"
 #include "executor/spi.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/fdwxact_launcher.h"
 #include "jit/jit.h"
 #include "libpq/libpq.h"
 #include "libpq/pqformat.h"
@@ -2904,6 +2906,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index c5ba149..d60d027 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -41,6 +41,7 @@
 #include "commands/vacuum.h"
 #include "commands/variable.h"
 #include "commands/trigger.h"
+#include "foreign/fdwxact.h"
 #include "funcapi.h"
 #include "jit/jit.h"
 #include "libpq/auth.h"
@@ -659,6 +660,10 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -1831,6 +1836,16 @@ static struct config_bool ConfigureNamesBool[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+			gettext_noop("Sets the usage of two-phase commit protocol for distributed transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		false,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
 	/* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL
@@ -2235,6 +2250,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index c0d3fb8..f398451 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -121,6 +121,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -287,6 +289,20 @@
 
 
 #------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
+#------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index ad06e8e..ca3eb62 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 3f203c6..dfecda1 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -208,6 +208,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdw_xact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 895a51f..5f0683d 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -306,6 +306,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_worker_processes);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_xacts setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index d63a3a2..e12cb00 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -730,6 +730,7 @@ GuessControlValues(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -957,6 +958,7 @@ RewriteControlFile(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* Contents are protected with a CRC */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca..15bfeb4 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -26,6 +26,7 @@
 #include "commands/dbcommands_xlog.h"
 #include "commands/sequence.h"
 #include "commands/tablespace.h"
+#include "foreign/fdwxact_xlog.h"
 #include "replication/message.h"
 #include "replication/origin.h"
 #include "rmgrdesc.h"
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 0bbe9879..c15dff7 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDW_XACT_ID, "Foreign Transactions", fdw_xact_redo, fdw_xact_desc, fdw_xact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 0e932da..b199c88 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 				TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index c7b4144..7180bd1 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -105,6 +105,13 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE				(1U << 3)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 30610b3..795e85a 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -227,6 +227,7 @@ typedef struct xl_parameter_change
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index 773d9e6..3d5333a 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -178,6 +178,7 @@ typedef struct ControlFileData
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index a146510..895337b 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5199,6 +5199,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '6053', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_fdwxact_resolver', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,oid,timestamptz}',
+  proargmodes => '{o,o,o,o}',
+  proargnames => '{pid,dbid,n_entries,last_resolved_time}',
+  prosrc => 'pg_stat_get_fdwxact_resolver' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5910,6 +5917,22 @@
   proargnames => '{type,name,args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '6050', descr => 'view foreign transactions',
+  proname => 'pg_prepared_fdw_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,text}',
+  proargmodes => '{o,o,o,o,o,o}',
+  proargnames => '{dbid,transaction,serverid,userid,status,identifier}',
+  prosrc => 'pg_prepared_fdw_xacts' },
+{ oid => '6051', descr => 'remove foreign transaction',
+  proname => 'pg_remove_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_remove_fdw_xact' },
+{ oid => '6052', descr => 'resolve foreign transaction',
+  proname => 'pg_resolve_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_resolve_fdw_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index c14eb54..f76e83d 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -13,6 +13,7 @@
 #define FDWAPI_H
 
 #include "access/parallel.h"
+#include "foreign/fdwxact.h"
 #include "nodes/execnodes.h"
 #include "nodes/relation.h"
 
@@ -168,6 +169,12 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef bool (*PrepareForeignTransaction_function) (ForeignTransaction *foreign_xact);
+typedef bool (*CommitForeignTransaction_function) (ForeignTransaction *foreign_xact);
+typedef bool (*RollbackForeignTransaction_function) (ForeignTransaction *foreing_xact);
+typedef bool (*ResolveForeignTransaction_function) (ForeignTransaction *foreign_xact,
+													bool is_commit);
+typedef bool (*IsTwoPhaseCommitEnabled_function) (Oid serverid);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -235,6 +242,13 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for distributed transactions */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	ResolveForeignTransaction_function ResolveForeignTransaction;
+	IsTwoPhaseCommitEnabled_function IsTwoPhaseCommitEnabled;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
@@ -247,7 +261,6 @@ typedef struct FdwRoutine
 	ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild;
 } FdwRoutine;
 
-
 /* Functions in foreign/foreign.c */
 extern FdwRoutine *GetFdwRoutine(Oid fdwhandler);
 extern Oid	GetForeignServerIdByRelId(Oid relid);
@@ -258,4 +271,7 @@ extern bool IsImportableForeignTable(const char *tablename,
 						 ImportForeignSchemaStmt *stmt);
 extern Path *GetExistingLocalJoinPath(RelOptInfo *joinrel);
 
+/* Functions in foreign/fdwxact.c */
+extern void FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, char *fx_id);
+
 #endif							/* FDWAPI_H */
diff --git a/src/include/foreign/fdwxact.h b/src/include/foreign/fdwxact.h
new file mode 100644
index 0000000..5138a2c
--- /dev/null
+++ b/src/include/foreign/fdwxact.h
@@ -0,0 +1,147 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL distributed transaction manager
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/foreign/fdwxact.h
+ */
+#ifndef FDW_XACT_H
+#define FDW_XACT_H
+
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "foreign/fdwxact_xlog.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+#define	FDW_XACT_NOT_WAITING		0
+#define	FDW_XACT_WAITING			1
+#define	FDW_XACT_WAIT_COMPLETE		2
+
+#define FdwXactEnabled() (max_prepared_foreign_xacts > 0)
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDW_XACT_ID_MAX_LEN 200
+
+/* Enum to track the status of prepared foreign transaction */
+typedef enum
+{
+	FDW_XACT_INITIAL,
+	FDW_XACT_PREPARING,					/* foreign transaction is being prepared */
+	FDW_XACT_PREPARED,					/* foriegn transaction is prepared */
+	FDW_XACT_COMMITTING_PREPARED,		/* foreign prepared transaction is to
+										 * be committed */
+	FDW_XACT_ABORTING_PREPARED, /* foreign prepared transaction is to be
+								 * aborted */
+} FdwXactStatus;
+
+/* Shared memory entry for a prepared or being prepared foreign transaction */
+typedef struct FdwXactData *FdwXact;
+
+typedef struct FdwXactData
+{
+	FdwXact		fxact_free_next;	/* Next free FdwXact entry */
+	FdwXact		fxact_next;		/* Pointer to the neext FdwXact entry accosiated
+								 * with the same transaction */
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	TransactionId local_xid;	/* XID of local transaction */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	FdwXactStatus status;		/* The state of the foreign
+								 * transaction. This doubles as the
+								 * action to be taken on this entry. */
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;		/* XLOG offset of inserting this entry start */
+	XLogRecPtr	insert_end_lsn;		/* XLOG offset of inserting this entry end */
+
+	bool		valid; /* Has the entry been complete and written to file? */
+	BackendId	registered_backend;	/* Backend who registered this entry */
+	bool		ondisk;			/* TRUE if prepare state file is on disk */
+	bool		inredo;			/* TRUE if entry was added via xlog_redo */
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN];		/* prepared transaction identifier */
+} FdwXactData;
+
+/* Shared memory layout for maintaining foreign prepared transaction entries. */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		freeFdwXacts;
+
+	/* Number of valid foreign transaction entries */
+	int			numFdwXacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdw_xacts[FLEXIBLE_ARRAY_MEMBER];		/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* Struct for foreign transaction resolution */
+typedef struct FdwXactResolveState
+{
+	Oid				dbid;		/* database oid */
+	TransactionId	wait_xid;	/* local transaction id waiting to be resolved */
+	PGPROC			*waiter;	/* backend process waiter */
+	FdwXact			fdwxact;	/* foreign transaction entries to resolve */
+} FdwXactResolveState;
+
+/* Struct for foreign transaction passed to API */
+typedef struct ForeignTransaction
+{
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	char			*fx_id;
+} ForeignTransaction;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern bool foreign_twophase_commit;
+
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern bool fdw_xact_exists(TransactionId xid, Oid dboid, Oid serverid,
+				Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern void PreCommit_FdwXacts(void);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern bool FdwXactResolveDistributedTransaction(FdwXactResolveState *fstate);
+extern void FdwXactResolveAllDanglingTransactions(Oid dbid);
+extern bool ForeignTwophaseCommitRequired(void);
+extern FdwXactResolveState *CreateFdwXactResolveState(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void FdwXactMarkForeignTransactionModified(ResultRelInfo *resultRelInfo,
+												  int flags);
+extern bool check_foreign_twophase_commit(bool *newval, void **extra,
+										  GucSource source);
+
+#endif   /* FDW_XACT_H */
diff --git a/src/include/foreign/fdwxact_launcher.h b/src/include/foreign/fdwxact_launcher.h
new file mode 100644
index 0000000..6ed003b
--- /dev/null
+++ b/src/include/foreign/fdwxact_launcher.h
@@ -0,0 +1,31 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/foreign/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _FDWXACT_LAUNCHER_H
+#define _FDWXACT_LAUNCHER_H
+
+#include "foreign/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherWakeup(void);
+
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+
+extern bool IsFdwXactLauncher(void);
+
+extern void fdwxact_maybe_launch_resolver(bool ignore_error);
+
+
+#endif	/* _FDWXACT_LAUNCHER_H */
diff --git a/src/include/foreign/fdwxact_resolver.h b/src/include/foreign/fdwxact_resolver.h
new file mode 100644
index 0000000..5afd98c
--- /dev/null
+++ b/src/include/foreign/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/foreign/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "foreign/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int foreign_xact_resolver_timeout;
+
+#endif		/* FDWXACT_RESOLVER_H */
diff --git a/src/include/foreign/fdwxact_xlog.h b/src/include/foreign/fdwxact_xlog.h
new file mode 100644
index 0000000..f42725e
--- /dev/null
+++ b/src/include/foreign/fdwxact_xlog.h
@@ -0,0 +1,51 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/foreign/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDW_XACT_INSERT	0x00
+#define XLOG_FDW_XACT_REMOVE	0x10
+
+/* Same as GIDSIZE */
+#define FDW_XACT_MAX_ID_LEN 200
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdw_xact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+} xl_fdw_xact_remove;
+
+extern void fdw_xact_redo(XLogReaderState *record);
+extern void fdw_xact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdw_xact_identify(uint8 info);
+
+#endif	/* FDWXACT_XLOG_H */
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 3ca12e6..d030368 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -68,10 +68,10 @@ typedef struct ForeignTable
 	List	   *options;		/* ftoptions as DefElem list */
 } ForeignTable;
 
-
 extern ForeignServer *GetForeignServer(Oid serverid);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperByName(const char *name,
 							bool missing_ok);
diff --git a/src/include/foreign/resolver_internal.h b/src/include/foreign/resolver_internal.h
new file mode 100644
index 0000000..9f8676b
--- /dev/null
+++ b/src/include/foreign/resolver_internal.h
@@ -0,0 +1,65 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/foreign/resovler_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _RESOLVER_INTERNAL_H
+#define _RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t	pid;	/* this resolver's PID, or 0 if not active */
+	Oid		dbid;	/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool	in_use;
+
+	/* Stats */
+	TimestampTz	last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t	mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	*latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/*
+	 * Foreign transaction resolution queue. Protected by FdwXactLock.
+	 */
+	SHM_QUEUE	FdwXactQueue;
+
+	/* Supervisor process */
+	pid_t		launcher_pid;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif	/* _RESOLVER_INTERNAL_H */
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index d59c24a..f74d1be 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -759,6 +759,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDW_XACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -832,7 +834,8 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDW_XACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -912,6 +915,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_READ,
+	WAIT_EVENT_FDW_XACT_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index 5c19a61..93953dc 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -150,6 +150,16 @@ struct PGPROC
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction
+								 * resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+
+	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
 	 * their lock.
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index 75bab29..25d6a2f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDW_XACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -124,4 +126,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 								TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 668d9ef..81560bd 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -94,6 +94,8 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 078129f..31502a0 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1413,6 +1413,13 @@ pg_policies| SELECT n.nspname AS schemaname,
    FROM ((pg_policy pol
      JOIN pg_class c ON ((c.oid = pol.polrelid)))
      LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace)));
+pg_prepared_fdw_xacts| SELECT f.dbid,
+    f.transaction,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.identifier
+   FROM pg_prepared_fdw_xacts() f(dbid, transaction, serverid, userid, status, identifier);
 pg_prepared_statements| SELECT p.name,
     p.statement,
     p.prepare_time,
@@ -1821,6 +1828,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_fdwxact_resolvers| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_fdwxact_resolver() r(pid, dbid, n_entries, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_progress_vacuum| SELECT s.pid,
     s.datid,
     d.datname,
-- 
2.10.5

From 7b99a4307c09c56a86556b67089b1f4fdd226b04 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:46:01 +0900
Subject: [PATCH v17 3/4] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/connection.c              | 534 +++++++++++++++++++------
 contrib/postgres_fdw/expected/postgres_fdw.out | 387 +++++++++++++++++-
 contrib/postgres_fdw/option.c                  |   5 +-
 contrib/postgres_fdw/postgres_fdw.c            |  60 ++-
 contrib/postgres_fdw/postgres_fdw.h            |  10 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql      | 151 ++++++-
 doc/src/sgml/postgres-fdw.sgml                 |  37 ++
 7 files changed, 1040 insertions(+), 144 deletions(-)

diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index fe4893a..9c0fa9a 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -15,8 +15,11 @@
 #include "postgres_fdw.h"
 
 #include "access/htup_details.h"
-#include "catalog/pg_user_mapping.h"
 #include "access/xact.h"
+#include "catalog/pg_user_mapping.h"
+#include "commands/defrem.h"
+#include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -56,6 +59,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		am_participant_of_ac;	/* true if fdwxact code control the transaction */
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -78,7 +82,7 @@ static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_xact_callback(XactEvent event, void *arg);
 static void pgfdw_subxact_callback(SubXactEvent event,
 					   SubTransactionId mySubid,
@@ -91,20 +95,14 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 						 bool ignore_errors);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 						 PGresult **result);
-
-
-/*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
- */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static bool pgfdw_commit_transaction(ConnCacheEntry *entry);
+static bool pgfdw_rollback_transaction(ConnCacheEntry *entry);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
 {
 	bool		found;
 	ConnCacheEntry *entry;
@@ -136,11 +134,8 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
 	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
+	key = umid;
 
 	/*
 	 * Find or create cached entry for requested connection.
@@ -182,6 +177,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping		*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +186,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->am_participant_of_ac = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +197,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,16 +213,46 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
 /*
  * Connect to remote server using specified server and user mapping properties.
+ * If the attempt to connect fails, and the caller can handle connection failure
+ * (connection_error_ok = true) return NULL, throw error otherwise.
  */
 static PGconn *
 connect_pg_server(ForeignServer *server, UserMapping *user)
@@ -265,11 +301,22 @@ connect_pg_server(ForeignServer *server, UserMapping *user)
 
 		conn = PQconnectdbParams(keywords, values, false);
 		if (!conn || PQstatus(conn) != CONNECTION_OK)
+		{
+			char	   *connmessage;
+			int			msglen;
+
+			/* libpq typically appends a newline, strip that */
+			connmessage = pstrdup(PQerrorMessage(conn));
+			msglen = strlen(connmessage);
+			if (msglen > 0 && connmessage[msglen - 1] == '\n')
+				connmessage[msglen - 1] = '\0';
+
 			ereport(ERROR,
 					(errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION),
 					 errmsg("could not connect to server \"%s\"",
 							server->servername),
 					 errdetail_internal("%s", pchomp(PQerrorMessage(conn)))));
+		}
 
 		/*
 		 * Check that non-superuser has used password to establish connection;
@@ -414,15 +461,24 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
+	ForeignServer	*server = GetForeignServer(serverid);
 
 	/* Start main transaction if we haven't yet */
 	if (entry->xact_depth <= 0)
 	{
 		const char *sql;
 
+		/* Register the new foreign server if enabled */
+		if (server_uses_twophase_commit(server))
+		{
+			/* Register foreign server with auto-generated identifer */
+			FdwXactRegisterForeignTransaction(serverid, userid, NULL);
+			entry->am_participant_of_ac = true;
+		}
+
 		elog(DEBUG3, "starting remote transaction on connection %p",
 			 entry->conn);
 
@@ -650,12 +706,11 @@ static void
 pgfdw_xact_callback(XactEvent event, void *arg)
 {
 	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
+	ConnCacheEntry	*entry;
 
-	/* Quick exit if no connections were touched in this transaction. */
+	/* Quick exit if no connections were touched in this transaction */
 	if (!xact_got_connection)
 		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote transactions, and
 	 * close them.
@@ -663,17 +718,20 @@ pgfdw_xact_callback(XactEvent event, void *arg)
 	hash_seq_init(&scan, ConnectionHash);
 	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
 	{
-		PGresult   *res;
-
 		/* Ignore cache entry if no open connection right now */
 		if (entry->conn == NULL)
 			continue;
 
+		/*
+		 * Foreign transactions participating to atomic commit are ended
+		 * by two-phase commit APIs. Ignore them.
+		 */
+		if (entry->am_participant_of_ac)
+			continue;
+
 		/* If it has an open remote transaction, try to close it */
 		if (entry->xact_depth > 0)
 		{
-			bool		abort_cleanup_failure = false;
-
 			elog(DEBUG3, "closing remote transaction on connection %p",
 				 entry->conn);
 
@@ -681,40 +739,7 @@ pgfdw_xact_callback(XactEvent event, void *arg)
 			{
 				case XACT_EVENT_PARALLEL_PRE_COMMIT:
 				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
+					pgfdw_commit_transaction(entry);
 					break;
 				case XACT_EVENT_PRE_PREPARE:
 
@@ -739,66 +764,7 @@ pgfdw_xact_callback(XactEvent event, void *arg)
 					break;
 				case XACT_EVENT_PARALLEL_ABORT:
 				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
+					pgfdw_rollback_transaction(entry);
 					break;
 			}
 		}
@@ -1193,3 +1159,325 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * The function prepares transaction on foreign server. This function
+ * is called only at the pre-commit phase of the local transaction. Since
+ * we should have the connection to the server that we are interested in
+ * we don't use serverid and userid that are necessary to get user mapping
+ * that is the key of the connection cache.
+ */
+bool
+postgresPrepareForeignTransaction(ForeignTransaction *foreign_xact)
+{
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+	PGresult	*res;
+	StringInfo	command;
+
+	entry = hash_search(ConnectionHash, &(foreign_xact->usermapping->umid),
+						HASH_FIND, NULL);
+
+	if (!entry->conn)
+		return false;
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", foreign_xact->fx_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	if (result)
+		elog(DEBUG1, "prepared foreign transaction on server %u with ID %s",
+			 foreign_xact->server->serverid, foreign_xact->fx_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/*
+ * The function commits the transactionon foreign server. This
+ * function is called both at the pre-commit phase of the local transaction
+ * when committing and at the end of the local transaction when aborting.
+ * Since we should the connections to the server that involved with the local
+ * transaction we don't use serverid and userid that are necessary to get
+ * user mapping that is the key of connection cache.
+ */
+bool
+postgresCommitForeignTransaction(ForeignTransaction *foreign_xact)
+{
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+
+	entry = hash_search(ConnectionHash, &(foreign_xact->usermapping->umid),
+						HASH_FIND, NULL);
+
+	result = pgfdw_commit_transaction(entry);
+
+	return result;
+}
+
+/*
+ * The function rollbacks the transactionon foreign server. This
+ * function is called both at the pre-commit phase of the local transaction
+ * when committing and at the end of the local transaction when aborting.
+ * Since we should the connections to the server that involved with the local
+ * transaction we don't use serverid and userid that are necessary to get
+ * user mapping that is the key of connection cache.
+ */
+bool
+postgresRollbackForeignTransaction(ForeignTransaction *foreign_xact)
+{
+	ConnCacheEntry *entry = NULL;
+	bool ret;
+
+	entry = hash_search(ConnectionHash, &(foreign_xact->usermapping->umid),
+						HASH_FIND, NULL);
+
+	/* Rollback a remote transaction */
+	ret = pgfdw_rollback_transaction(entry);
+
+	return ret;
+}
+
+bool
+postgresResolveForeignTransaction(ForeignTransaction *foreign_xact, bool is_commit)
+{
+	ConnCacheEntry *entry = NULL;
+	StringInfo	command;
+	bool result;
+	PGresult	*res;
+
+	entry = GetConnectionState(foreign_xact->usermapping->umid,
+							   false, false);
+
+	if (!entry->conn)
+		return false;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 foreign_xact->fx_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		/*
+		 * The command failed, raise a warning to log the reason of failure.
+		 * We may not be in a transaction here, so raising error doesn't
+		 * help. Even if we are in a transaction, it would be the resolver
+		 * transaction, which will get aborted on raising error, thus
+		 * delaying resolution of other prepared foreign transactions.
+		 */
+		pgfdw_report_error(WARNING, res, entry->conn, false, command->data);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * If we tried to COMMIT/ABORT a prepared transaction and the prepared
+		 * transaction was missing on the foreign server, it was probably
+		 * resolved by some other means. Anyway, it should be considered as resolved.
+		 */
+		result = (sqlstate == ERRCODE_UNDEFINED_OBJECT);
+	}
+	else
+		result = true;
+
+	elog(DEBUG1, "%s prepared foreign transaction on server %u with ID %s",
+		 is_commit ? "commit" : "rollback", foreign_xact->server->serverid,
+		 foreign_xact->fx_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->am_participant_of_ac = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	/*
+	 * Regardless of the event type, we can now mark ourselves as out of the
+	 * transaction.
+	 */
+	xact_got_connection = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
+
+static bool
+pgfdw_rollback_transaction(ConnCacheEntry *entry)
+{
+	bool abort_cleanup_failure = false;
+
+	/*
+	 * In rollback local transaction, if we don't the connection
+	 * it means any transaction started. So we can ragard it as
+	 * success.
+	 */
+	if (!entry || !entry->conn)
+		return true;
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is already unsalvageable, don't touch it
+	 * further.
+	 */
+	if (entry->changing_xact_state)
+		return true;
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+	else
+	{
+		entry->have_prep_stmt = false;
+		entry->have_error = false;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return !abort_cleanup_failure;
+}
+
+static bool
+pgfdw_commit_transaction(ConnCacheEntry *entry)
+{
+	PGresult	*res;
+	bool result = false;
+
+	if (!entry || !entry->conn)
+		return false;
+
+	/*
+	 * If abort cleanup previously failed for this connection,
+	 * we can't issue any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	/*
+	 * If there were any errors in subtransactions, and we
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+	entry->have_prep_stmt = false;
+	entry->have_error = false;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index f5498c6..933d88b 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,15 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -185,16 +207,19 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                                      List of foreign tables
- Schema |   Table    |  Server   |                   FDW options                    | Description 
---------+------------+-----------+--------------------------------------------------+-------------
- public | ft1        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft2        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft4        | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
- public | ft5        | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft6        | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft_pg_type | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
-(6 rows)
+                                         List of foreign tables
+ Schema |      Table       |  Server   |                   FDW options                    | Description 
+--------+------------------+-----------+--------------------------------------------------+-------------
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
+(9 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8607,3 +8632,345 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+\det+
+                                                 List of foreign tables
+ Schema |      Table       |  Server   |                            FDW options                            | Description 
+--------+------------------+-----------+-------------------------------------------------------------------+-------------
+ public | fpagg_tab_p1     | loopback  | (table_name 'pagg_tab_p1')                                        | 
+ public | fpagg_tab_p2     | loopback  | (table_name 'pagg_tab_p2')                                        | 
+ public | fpagg_tab_p3     | loopback  | (table_name 'pagg_tab_p3')                                        | 
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')                             | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1', use_remote_estimate 'true') | 
+ public | ft3              | loopback  | (table_name 'loct3', use_remote_estimate 'true')                  | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')                             | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type')                  | 
+ public | ftprt1_p1        | loopback  | (table_name 'fprt1_p1', use_remote_estimate 'true')               | 
+ public | ftprt1_p2        | loopback  | (table_name 'fprt1_p2')                                           | 
+ public | ftprt2_p1        | loopback  | (table_name 'fprt2_p1', use_remote_estimate 'true')               | 
+ public | ftprt2_p2        | loopback  | (table_name 'fprt2_p2', use_remote_estimate 'true')               | 
+ public | rem1             | loopback  | (table_name 'loc1')                                               | 
+ public | rem2             | loopback  | (table_name 'loc2')                                               | 
+(19 rows)
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+  srvname  
+-----------
+ loopback
+ loopback2
+ loopback3
+(3 rows)
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO on;
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO ft8_twophase VALUES(3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO "S 1"."T 6" VALUES (4);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  4
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(5);
+INSERT INTO "S 1"."T 6" VALUES (5);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  4
+(1 row)
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(8);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Fails, cannot commit the distributed transaction if 2PC-non-capable
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+ERROR:  cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Disables atomic commit, and success the same case as above.
+SET foreign_twophase_commit TO off;
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+(5 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+(5 rows)
+
+-- Enable atomic commit, again.
+SET foreign_twophase_commit TO on;
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(10);
+INSERT INTO ft8_twophase VALUES(10);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+-- Fails, cannot prepare the transaction if non-supporeted
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(11);
+INSERT INTO ft9_not_twophase VALUES(11);
+PREPARE TRANSACTION 'gx1'; -- error
+ERROR:  cannot prepare a transaction that modified remote tables
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
diff --git a/contrib/postgres_fdw/option.c b/contrib/postgres_fdw/option.c
index 6854f1b..1f45b1c 100644
--- a/contrib/postgres_fdw/option.c
+++ b/contrib/postgres_fdw/option.c
@@ -108,7 +108,8 @@ postgres_fdw_validator(PG_FUNCTION_ARGS)
 		 * Validate option value, when we can do so without any context.
 		 */
 		if (strcmp(def->defname, "use_remote_estimate") == 0 ||
-			strcmp(def->defname, "updatable") == 0)
+			strcmp(def->defname, "updatable") == 0 ||
+			strcmp(def->defname, "two_phase_commit") == 0)
 		{
 			/* these accept only boolean values */
 			(void) defGetBoolean(def);
@@ -177,6 +178,8 @@ InitPgFdwOptions(void)
 		/* fetch_size is available on both server and table */
 		{"fetch_size", ForeignServerRelationId, false},
 		{"fetch_size", ForeignTableRelationId, false},
+		/* two phase commit support */
+		{"two_phase_commit", ForeignServerRelationId, false},
 		{NULL, InvalidOid, false}
 	};
 
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 0803c30..24191a2 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include "postgres_fdw.h"
 
+#include "access/xact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "catalog/pg_class.h"
@@ -21,6 +22,7 @@
 #include "commands/explain.h"
 #include "commands/vacuum.h"
 #include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
 #include "funcapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -359,6 +361,7 @@ static void postgresGetForeignUpperPaths(PlannerInfo *root,
 							 RelOptInfo *input_rel,
 							 RelOptInfo *output_rel,
 							 void *extra);
+static bool postgresIsTwoPhaseCommitEnabled(Oid serverid);
 
 /*
  * Helper functions
@@ -452,7 +455,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 				  const PgFdwRelationInfo *fpinfo_o,
 				  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -506,10 +508,29 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+	routine->ResolveForeignTransaction = postgresResolveForeignTransaction;
+	routine->IsTwoPhaseCommitEnabled = postgresIsTwoPhaseCommitEnabled;
+
 	PG_RETURN_POINTER(routine);
 }
 
 /*
+ * postgresIsTwoPhaseCommitEnabled
+ */
+static bool
+postgresIsTwoPhaseCommitEnabled(Oid serverid)
+{
+	ForeignServer	*server = GetForeignServer(serverid);
+
+
+	return server_uses_twophase_commit(server);
+}
+
+/*
  * postgresGetForeignRelSize
  *		Estimate # of rows and width of the result of the scan
  *
@@ -1356,7 +1377,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2412,7 +2433,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2709,7 +2730,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								&retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3326,7 +3347,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4113,7 +4134,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4203,7 +4224,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4426,7 +4447,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
@@ -5807,3 +5828,26 @@ find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel)
 	/* We didn't find any suitable equivalence class expression */
 	return NULL;
 }
+
+/*
+ * server_uses_twophase_commit
+ * Returns true if the foreign server is configured to support 2PC.
+ */
+bool
+server_uses_twophase_commit(ForeignServer *server)
+{
+	ListCell		*lc;
+
+	/* Check the options for two phase compliance */
+	foreach(lc, server->options)
+	{
+		DefElem    *d = (DefElem *) lfirst(lc);
+
+		if (strcmp(d->defname, "two_phase_commit") == 0)
+		{
+			return defGetBoolean(d);
+		}
+	}
+	/* By default a server is not 2PC compliant */
+	return false;
+}
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index 70b538e..4d1b754 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "foreign/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/relation.h"
@@ -115,7 +116,8 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
+extern PGconn *GetExistingConnection(Oid umid);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -123,6 +125,11 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 				   bool clear, const char *sql);
+extern bool postgresPrepareForeignTransaction(ForeignTransaction *foreign_xact);
+extern bool postgresCommitForeignTransaction(ForeignTransaction *foreign_xact);
+extern bool postgresRollbackForeignTransaction(ForeignTransaction *foriegn_xact);
+extern bool postgresResolveForeignTransaction(ForeignTransaction *foreign_xact,
+											  bool is_commit);
 
 /* in option.c */
 extern int ExtractConnectionOptions(List *defelems,
@@ -181,6 +188,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 						List *remote_conds, List *pathkeys, bool is_subquery,
 						List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index e1b955f..9bbd159 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,19 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -2298,7 +2325,6 @@ SELECT t1.a, t1.phv, t2.b, t2.phv FROM (SELECT 't1_phv' phv, * FROM fprt1 WHERE
 
 RESET enable_partitionwise_join;
 
-
 -- ===================================================================
 -- test partitionwise aggregates
 -- ===================================================================
@@ -2348,3 +2374,126 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+
+\det+
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO on;
+
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO ft8_twophase VALUES(3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO "S 1"."T 6" VALUES (4);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(5);
+INSERT INTO "S 1"."T 6" VALUES (5);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(8);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Fails, cannot commit the distributed transaction if 2PC-non-capable
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Disables atomic commit, and success the same case as above.
+SET foreign_twophase_commit TO off;
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Enable atomic commit, again.
+SET foreign_twophase_commit TO on;
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(10);
+INSERT INTO ft8_twophase VALUES(10);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Fails, cannot prepare the transaction if non-supporeted
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(11);
+INSERT INTO ft9_not_twophase VALUES(11);
+PREPARE TRANSACTION 'gx1'; -- error
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 54b5e98..f4a9ff5 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -436,6 +436,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
-- 
2.10.5

From 1ce2f489cb29e2ea86298e4ddae85f8d2ed38b28 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:48:08 +0900
Subject: [PATCH v17 4/4] Add regression tests for atomic commit.

---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index daf79a0..71c8b9d 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000..a23f120
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port', two_phase_commit 'on');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port', two_phase_commit 'on');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_prepared_fdw_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 2ff2acc..bfc8f53 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2286,9 +2286,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2303,7 +2306,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.10.5

From c655515f56192080e0aef9b138952ae33713c091 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Thu, 8 Feb 2018 11:26:46 +0900
Subject: [PATCH v18 1/4] Keep track of writing on non-temporary relation.

---
 src/backend/access/heap/heapam.c | 12 ++++++++++++
 src/include/access/xact.h        |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 5f1a69c..31a44ca 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2623,6 +2623,10 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 		heap_freetuple(heaptup);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleGetOid(tup);
 }
 
@@ -3444,6 +3448,10 @@ l1:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleMayBeUpdated;
 }
 
@@ -4394,6 +4402,10 @@ l2:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	bms_free(hot_attrs);
 	bms_free(proj_idx_attrs);
 	bms_free(key_attrs);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 083e879..c7b4144 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -98,6 +98,11 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
-- 
2.10.5

From 0bdaab36e322809a393cbae052dc1da8a5599c1e Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:46:01 +0900
Subject: [PATCH v18 3/4] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/connection.c              | 534 +++++++++++++++++++------
 contrib/postgres_fdw/expected/postgres_fdw.out | 387 +++++++++++++++++-
 contrib/postgres_fdw/option.c                  |   5 +-
 contrib/postgres_fdw/postgres_fdw.c            |  60 ++-
 contrib/postgres_fdw/postgres_fdw.h            |  10 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql      | 151 ++++++-
 doc/src/sgml/postgres-fdw.sgml                 |  37 ++
 7 files changed, 1040 insertions(+), 144 deletions(-)

diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index fe4893a..9c0fa9a 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -15,8 +15,11 @@
 #include "postgres_fdw.h"
 
 #include "access/htup_details.h"
-#include "catalog/pg_user_mapping.h"
 #include "access/xact.h"
+#include "catalog/pg_user_mapping.h"
+#include "commands/defrem.h"
+#include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -56,6 +59,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		am_participant_of_ac;	/* true if fdwxact code control the transaction */
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -78,7 +82,7 @@ static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_xact_callback(XactEvent event, void *arg);
 static void pgfdw_subxact_callback(SubXactEvent event,
 					   SubTransactionId mySubid,
@@ -91,20 +95,14 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 						 bool ignore_errors);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 						 PGresult **result);
-
-
-/*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
- */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static bool pgfdw_commit_transaction(ConnCacheEntry *entry);
+static bool pgfdw_rollback_transaction(ConnCacheEntry *entry);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
 {
 	bool		found;
 	ConnCacheEntry *entry;
@@ -136,11 +134,8 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
 	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
+	key = umid;
 
 	/*
 	 * Find or create cached entry for requested connection.
@@ -182,6 +177,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping		*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +186,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->am_participant_of_ac = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +197,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,16 +213,46 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
 /*
  * Connect to remote server using specified server and user mapping properties.
+ * If the attempt to connect fails, and the caller can handle connection failure
+ * (connection_error_ok = true) return NULL, throw error otherwise.
  */
 static PGconn *
 connect_pg_server(ForeignServer *server, UserMapping *user)
@@ -265,11 +301,22 @@ connect_pg_server(ForeignServer *server, UserMapping *user)
 
 		conn = PQconnectdbParams(keywords, values, false);
 		if (!conn || PQstatus(conn) != CONNECTION_OK)
+		{
+			char	   *connmessage;
+			int			msglen;
+
+			/* libpq typically appends a newline, strip that */
+			connmessage = pstrdup(PQerrorMessage(conn));
+			msglen = strlen(connmessage);
+			if (msglen > 0 && connmessage[msglen - 1] == '\n')
+				connmessage[msglen - 1] = '\0';
+
 			ereport(ERROR,
 					(errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION),
 					 errmsg("could not connect to server \"%s\"",
 							server->servername),
 					 errdetail_internal("%s", pchomp(PQerrorMessage(conn)))));
+		}
 
 		/*
 		 * Check that non-superuser has used password to establish connection;
@@ -414,15 +461,24 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
+	ForeignServer	*server = GetForeignServer(serverid);
 
 	/* Start main transaction if we haven't yet */
 	if (entry->xact_depth <= 0)
 	{
 		const char *sql;
 
+		/* Register the new foreign server if enabled */
+		if (server_uses_twophase_commit(server))
+		{
+			/* Register foreign server with auto-generated identifer */
+			FdwXactRegisterForeignTransaction(serverid, userid, NULL);
+			entry->am_participant_of_ac = true;
+		}
+
 		elog(DEBUG3, "starting remote transaction on connection %p",
 			 entry->conn);
 
@@ -650,12 +706,11 @@ static void
 pgfdw_xact_callback(XactEvent event, void *arg)
 {
 	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
+	ConnCacheEntry	*entry;
 
-	/* Quick exit if no connections were touched in this transaction. */
+	/* Quick exit if no connections were touched in this transaction */
 	if (!xact_got_connection)
 		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote transactions, and
 	 * close them.
@@ -663,17 +718,20 @@ pgfdw_xact_callback(XactEvent event, void *arg)
 	hash_seq_init(&scan, ConnectionHash);
 	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
 	{
-		PGresult   *res;
-
 		/* Ignore cache entry if no open connection right now */
 		if (entry->conn == NULL)
 			continue;
 
+		/*
+		 * Foreign transactions participating to atomic commit are ended
+		 * by two-phase commit APIs. Ignore them.
+		 */
+		if (entry->am_participant_of_ac)
+			continue;
+
 		/* If it has an open remote transaction, try to close it */
 		if (entry->xact_depth > 0)
 		{
-			bool		abort_cleanup_failure = false;
-
 			elog(DEBUG3, "closing remote transaction on connection %p",
 				 entry->conn);
 
@@ -681,40 +739,7 @@ pgfdw_xact_callback(XactEvent event, void *arg)
 			{
 				case XACT_EVENT_PARALLEL_PRE_COMMIT:
 				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
+					pgfdw_commit_transaction(entry);
 					break;
 				case XACT_EVENT_PRE_PREPARE:
 
@@ -739,66 +764,7 @@ pgfdw_xact_callback(XactEvent event, void *arg)
 					break;
 				case XACT_EVENT_PARALLEL_ABORT:
 				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
+					pgfdw_rollback_transaction(entry);
 					break;
 			}
 		}
@@ -1193,3 +1159,325 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * The function prepares transaction on foreign server. This function
+ * is called only at the pre-commit phase of the local transaction. Since
+ * we should have the connection to the server that we are interested in
+ * we don't use serverid and userid that are necessary to get user mapping
+ * that is the key of the connection cache.
+ */
+bool
+postgresPrepareForeignTransaction(ForeignTransaction *foreign_xact)
+{
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+	PGresult	*res;
+	StringInfo	command;
+
+	entry = hash_search(ConnectionHash, &(foreign_xact->usermapping->umid),
+						HASH_FIND, NULL);
+
+	if (!entry->conn)
+		return false;
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", foreign_xact->fx_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	if (result)
+		elog(DEBUG1, "prepared foreign transaction on server %u with ID %s",
+			 foreign_xact->server->serverid, foreign_xact->fx_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/*
+ * The function commits the transactionon foreign server. This
+ * function is called both at the pre-commit phase of the local transaction
+ * when committing and at the end of the local transaction when aborting.
+ * Since we should the connections to the server that involved with the local
+ * transaction we don't use serverid and userid that are necessary to get
+ * user mapping that is the key of connection cache.
+ */
+bool
+postgresCommitForeignTransaction(ForeignTransaction *foreign_xact)
+{
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+
+	entry = hash_search(ConnectionHash, &(foreign_xact->usermapping->umid),
+						HASH_FIND, NULL);
+
+	result = pgfdw_commit_transaction(entry);
+
+	return result;
+}
+
+/*
+ * The function rollbacks the transactionon foreign server. This
+ * function is called both at the pre-commit phase of the local transaction
+ * when committing and at the end of the local transaction when aborting.
+ * Since we should the connections to the server that involved with the local
+ * transaction we don't use serverid and userid that are necessary to get
+ * user mapping that is the key of connection cache.
+ */
+bool
+postgresRollbackForeignTransaction(ForeignTransaction *foreign_xact)
+{
+	ConnCacheEntry *entry = NULL;
+	bool ret;
+
+	entry = hash_search(ConnectionHash, &(foreign_xact->usermapping->umid),
+						HASH_FIND, NULL);
+
+	/* Rollback a remote transaction */
+	ret = pgfdw_rollback_transaction(entry);
+
+	return ret;
+}
+
+bool
+postgresResolveForeignTransaction(ForeignTransaction *foreign_xact, bool is_commit)
+{
+	ConnCacheEntry *entry = NULL;
+	StringInfo	command;
+	bool result;
+	PGresult	*res;
+
+	entry = GetConnectionState(foreign_xact->usermapping->umid,
+							   false, false);
+
+	if (!entry->conn)
+		return false;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 foreign_xact->fx_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		/*
+		 * The command failed, raise a warning to log the reason of failure.
+		 * We may not be in a transaction here, so raising error doesn't
+		 * help. Even if we are in a transaction, it would be the resolver
+		 * transaction, which will get aborted on raising error, thus
+		 * delaying resolution of other prepared foreign transactions.
+		 */
+		pgfdw_report_error(WARNING, res, entry->conn, false, command->data);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * If we tried to COMMIT/ABORT a prepared transaction and the prepared
+		 * transaction was missing on the foreign server, it was probably
+		 * resolved by some other means. Anyway, it should be considered as resolved.
+		 */
+		result = (sqlstate == ERRCODE_UNDEFINED_OBJECT);
+	}
+	else
+		result = true;
+
+	elog(DEBUG1, "%s prepared foreign transaction on server %u with ID %s",
+		 is_commit ? "commit" : "rollback", foreign_xact->server->serverid,
+		 foreign_xact->fx_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->am_participant_of_ac = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	/*
+	 * Regardless of the event type, we can now mark ourselves as out of the
+	 * transaction.
+	 */
+	xact_got_connection = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
+
+static bool
+pgfdw_rollback_transaction(ConnCacheEntry *entry)
+{
+	bool abort_cleanup_failure = false;
+
+	/*
+	 * In rollback local transaction, if we don't the connection
+	 * it means any transaction started. So we can ragard it as
+	 * success.
+	 */
+	if (!entry || !entry->conn)
+		return true;
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is already unsalvageable, don't touch it
+	 * further.
+	 */
+	if (entry->changing_xact_state)
+		return true;
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+	else
+	{
+		entry->have_prep_stmt = false;
+		entry->have_error = false;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return !abort_cleanup_failure;
+}
+
+static bool
+pgfdw_commit_transaction(ConnCacheEntry *entry)
+{
+	PGresult	*res;
+	bool result = false;
+
+	if (!entry || !entry->conn)
+		return false;
+
+	/*
+	 * If abort cleanup previously failed for this connection,
+	 * we can't issue any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	/*
+	 * If there were any errors in subtransactions, and we
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+	entry->have_prep_stmt = false;
+	entry->have_error = false;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 21a2ef5..15dadf4 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,15 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -185,16 +207,19 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                                      List of foreign tables
- Schema |   Table    |  Server   |                   FDW options                    | Description 
---------+------------+-----------+--------------------------------------------------+-------------
- public | ft1        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft2        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft4        | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
- public | ft5        | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft6        | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft_pg_type | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
-(6 rows)
+                                         List of foreign tables
+ Schema |      Table       |  Server   |                   FDW options                    | Description 
+--------+------------------+-----------+--------------------------------------------------+-------------
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
+(9 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8650,3 +8675,345 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+\det+
+                                                 List of foreign tables
+ Schema |      Table       |  Server   |                            FDW options                            | Description 
+--------+------------------+-----------+-------------------------------------------------------------------+-------------
+ public | fpagg_tab_p1     | loopback  | (table_name 'pagg_tab_p1')                                        | 
+ public | fpagg_tab_p2     | loopback  | (table_name 'pagg_tab_p2')                                        | 
+ public | fpagg_tab_p3     | loopback  | (table_name 'pagg_tab_p3')                                        | 
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')                             | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1', use_remote_estimate 'true') | 
+ public | ft3              | loopback  | (table_name 'loct3', use_remote_estimate 'true')                  | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')                             | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type')                  | 
+ public | ftprt1_p1        | loopback  | (table_name 'fprt1_p1', use_remote_estimate 'true')               | 
+ public | ftprt1_p2        | loopback  | (table_name 'fprt1_p2')                                           | 
+ public | ftprt2_p1        | loopback  | (table_name 'fprt2_p1', use_remote_estimate 'true')               | 
+ public | ftprt2_p2        | loopback  | (table_name 'fprt2_p2', use_remote_estimate 'true')               | 
+ public | rem1             | loopback  | (table_name 'loc1')                                               | 
+ public | rem2             | loopback  | (table_name 'loc2')                                               | 
+(19 rows)
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+  srvname  
+-----------
+ loopback
+ loopback2
+ loopback3
+(3 rows)
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO on;
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO ft8_twophase VALUES(3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO "S 1"."T 6" VALUES (4);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  4
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(5);
+INSERT INTO "S 1"."T 6" VALUES (5);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  4
+(1 row)
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(8);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Fails, cannot commit the distributed transaction if 2PC-non-capable
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+ERROR:  cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Disables atomic commit, and success the same case as above.
+SET foreign_twophase_commit TO off;
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+(5 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+(5 rows)
+
+-- Enable atomic commit, again.
+SET foreign_twophase_commit TO on;
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(10);
+INSERT INTO ft8_twophase VALUES(10);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+-- Fails, cannot prepare the transaction if non-supporeted
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(11);
+INSERT INTO ft9_not_twophase VALUES(11);
+PREPARE TRANSACTION 'gx1'; -- error
+ERROR:  cannot prepare a transaction that modified remote tables
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
diff --git a/contrib/postgres_fdw/option.c b/contrib/postgres_fdw/option.c
index 6854f1b..1f45b1c 100644
--- a/contrib/postgres_fdw/option.c
+++ b/contrib/postgres_fdw/option.c
@@ -108,7 +108,8 @@ postgres_fdw_validator(PG_FUNCTION_ARGS)
 		 * Validate option value, when we can do so without any context.
 		 */
 		if (strcmp(def->defname, "use_remote_estimate") == 0 ||
-			strcmp(def->defname, "updatable") == 0)
+			strcmp(def->defname, "updatable") == 0 ||
+			strcmp(def->defname, "two_phase_commit") == 0)
 		{
 			/* these accept only boolean values */
 			(void) defGetBoolean(def);
@@ -177,6 +178,8 @@ InitPgFdwOptions(void)
 		/* fetch_size is available on both server and table */
 		{"fetch_size", ForeignServerRelationId, false},
 		{"fetch_size", ForeignTableRelationId, false},
+		/* two phase commit support */
+		{"two_phase_commit", ForeignServerRelationId, false},
 		{NULL, InvalidOid, false}
 	};
 
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 6cbba97..20365a4 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include "postgres_fdw.h"
 
+#include "access/xact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "catalog/pg_class.h"
@@ -21,6 +22,7 @@
 #include "commands/explain.h"
 #include "commands/vacuum.h"
 #include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
 #include "funcapi.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -359,6 +361,7 @@ static void postgresGetForeignUpperPaths(PlannerInfo *root,
 							 RelOptInfo *input_rel,
 							 RelOptInfo *output_rel,
 							 void *extra);
+static bool postgresIsTwoPhaseCommitEnabled(Oid serverid);
 
 /*
  * Helper functions
@@ -452,7 +455,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 				  const PgFdwRelationInfo *fpinfo_o,
 				  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -506,10 +508,29 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+	routine->ResolveForeignTransaction = postgresResolveForeignTransaction;
+	routine->IsTwoPhaseCommitEnabled = postgresIsTwoPhaseCommitEnabled;
+
 	PG_RETURN_POINTER(routine);
 }
 
 /*
+ * postgresIsTwoPhaseCommitEnabled
+ */
+static bool
+postgresIsTwoPhaseCommitEnabled(Oid serverid)
+{
+	ForeignServer	*server = GetForeignServer(serverid);
+
+
+	return server_uses_twophase_commit(server);
+}
+
+/*
  * postgresGetForeignRelSize
  *		Estimate # of rows and width of the result of the scan
  *
@@ -1356,7 +1377,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2411,7 +2432,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2708,7 +2729,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								&retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3325,7 +3346,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4112,7 +4133,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4202,7 +4223,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4425,7 +4446,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
@@ -5807,3 +5828,26 @@ find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel)
 	/* We didn't find any suitable equivalence class expression */
 	return NULL;
 }
+
+/*
+ * server_uses_twophase_commit
+ * Returns true if the foreign server is configured to support 2PC.
+ */
+bool
+server_uses_twophase_commit(ForeignServer *server)
+{
+	ListCell		*lc;
+
+	/* Check the options for two phase compliance */
+	foreach(lc, server->options)
+	{
+		DefElem    *d = (DefElem *) lfirst(lc);
+
+		if (strcmp(d->defname, "two_phase_commit") == 0)
+		{
+			return defGetBoolean(d);
+		}
+	}
+	/* By default a server is not 2PC compliant */
+	return false;
+}
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index 70b538e..4d1b754 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "foreign/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/relation.h"
@@ -115,7 +116,8 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
+extern PGconn *GetExistingConnection(Oid umid);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -123,6 +125,11 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 				   bool clear, const char *sql);
+extern bool postgresPrepareForeignTransaction(ForeignTransaction *foreign_xact);
+extern bool postgresCommitForeignTransaction(ForeignTransaction *foreign_xact);
+extern bool postgresRollbackForeignTransaction(ForeignTransaction *foriegn_xact);
+extern bool postgresResolveForeignTransaction(ForeignTransaction *foreign_xact,
+											  bool is_commit);
 
 /* in option.c */
 extern int ExtractConnectionOptions(List *defelems,
@@ -181,6 +188,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 						List *remote_conds, List *pathkeys, bool is_subquery,
 						List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 88c4cb4..2554c9c 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,19 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -2304,7 +2331,6 @@ SELECT t1.a, t2.b FROM fprt1 t1 INNER JOIN fprt2 t2 ON (t1.a = t2.b) WHERE t1.a
 
 RESET enable_partitionwise_join;
 
-
 -- ===================================================================
 -- test partitionwise aggregates
 -- ===================================================================
@@ -2354,3 +2380,126 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+
+\det+
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO on;
+
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO ft8_twophase VALUES(3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO "S 1"."T 6" VALUES (4);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(5);
+INSERT INTO "S 1"."T 6" VALUES (5);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(8);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Fails, cannot commit the distributed transaction if 2PC-non-capable
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Disables atomic commit, and success the same case as above.
+SET foreign_twophase_commit TO off;
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Enable atomic commit, again.
+SET foreign_twophase_commit TO on;
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(10);
+INSERT INTO ft8_twophase VALUES(10);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Fails, cannot prepare the transaction if non-supporeted
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(11);
+INSERT INTO ft9_not_twophase VALUES(11);
+PREPARE TRANSACTION 'gx1'; -- error
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 54b5e98..f4a9ff5 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -436,6 +436,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
-- 
2.10.5

From 2b93b86357c8456273262738a1da72cdaa68f138 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:48:08 +0900
Subject: [PATCH v18 4/4] Add regression tests for atomic commit.

---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index daf79a0..71c8b9d 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000..a23f120
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port', two_phase_commit 'on');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port', two_phase_commit 'on');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_prepared_fdw_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 6890678..d1b181a 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2286,9 +2286,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2303,7 +2306,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.10.5

From b5cb1921a90f7b0f08b7c47119f1b7524e2b6edd Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:44:42 +0900
Subject: [PATCH v18 2/4] Support atomic commit among multiple foreign servers.

---
 doc/src/sgml/catalogs.sgml                    |   97 +
 doc/src/sgml/config.sgml                      |  124 ++
 doc/src/sgml/fdwhandler.sgml                  |  200 ++
 doc/src/sgml/func.sgml                        |   51 +
 doc/src/sgml/monitoring.sgml                  |   56 +
 src/backend/access/rmgrdesc/Makefile          |    8 +-
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   65 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/Makefile           |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   32 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/foreigncmds.c            |   23 +
 src/backend/executor/execPartition.c          |    4 +
 src/backend/executor/nodeForeignscan.c        |    8 +
 src/backend/executor/nodeModifyTable.c        |    5 +
 src/backend/foreign/Makefile                  |    2 +-
 src/backend/foreign/fdwxact.c                 | 2762 +++++++++++++++++++++++++
 src/backend/foreign/fdwxact_launcher.c        |  587 ++++++
 src/backend/foreign/fdwxact_resolver.c        |  310 +++
 src/backend/foreign/foreign.c                 |   43 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   18 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    5 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    2 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   61 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   23 +
 src/include/foreign/fdwapi.h                  |   18 +-
 src/include/foreign/fdwxact.h                 |  147 ++
 src/include/foreign/fdwxact_launcher.h        |   31 +
 src/include/foreign/fdwxact_resolver.h        |   23 +
 src/include/foreign/fdwxact_xlog.h            |   51 +
 src/include/foreign/foreign.h                 |    2 +-
 src/include/foreign/resolver_internal.h       |   65 +
 src/include/pgstat.h                          |    8 +-
 src/include/storage/proc.h                    |   10 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    2 +
 src/test/regress/expected/rules.out           |   12 +
 57 files changed, 5052 insertions(+), 27 deletions(-)
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 100755 src/backend/foreign/fdwxact.c
 create mode 100644 src/backend/foreign/fdwxact_launcher.c
 create mode 100644 src/backend/foreign/fdwxact_resolver.c
 create mode 100644 src/include/foreign/fdwxact.h
 create mode 100644 src/include/foreign/fdwxact_launcher.h
 create mode 100644 src/include/foreign/fdwxact_resolver.h
 create mode 100644 src/include/foreign/fdwxact_xlog.h
 create mode 100644 src/include/foreign/resolver_internal.h

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 0179dee..792f361 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9622,6 +9622,103 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-prepared-fdw-xacts">
+  <title><structname>pg_prepared_fdw_xacts</structname></title>
+
+  <indexterm zone="view-pg-prepared-fdw-xacts">
+   <primary>pg_prepared_fdw_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_prepared_fdw_xacts</structname> displays
+   information about foreign transactions that are currently prepared on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="fdw-transaction-managements"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_prepared_xacts</structname> contains one row per prepared
+   foreign transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_prepared_fdw_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>transaction</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Transaction id that this foreign transaction associates with
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server that this foreign server is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction: <literal>prepared</literal>, <literal>committing</literal>, <literal>aborting</literal> or <literal>unknown</literal>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index f11b8f7..406fd9c 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1546,6 +1546,29 @@ include_dir 'conf.d'
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+      <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Sets the maximum number of foreign transactions that can be prepared
+        simultaneously. A single local transaction can give rise to multiple
+        foreign transaction. If <literal>N</literal> local transactions each
+        across <literal>K</literal> foreign server this value need to be set
+        <literal>N * K</literal>, not just <literal>N</literal>.
+        This parameter can only be set at server start.
+       </para>
+       <para>
+        When running a standby server, you must set this parameter to the
+        same or higher value than on the master server. Otherwise, queries
+        will not be allowed in the standby server.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-work-mem" xreflabel="work_mem">
       <term><varname>work_mem</varname> (<type>integer</type>)
       <indexterm>
@@ -3611,6 +3634,78 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
      </variablelist>
     </sect2>
 
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+
+     <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+      <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+      <indexterm>
+       <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+        resolver is responsible for foreign transaction resolution on one database.
+       </para>
+       <para>
+        Foreign transaction resolution workers are taken from the pool defined by
+        <varname>max_worker_processes</varname>.
+       </para>
+       <para>
+        The default value is 0.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry id="foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+      <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specify how long the foreign transaction resolver should wait when the last resolution
+        fails before retrying to resolve foreign transaction. This parameter can only be set in the
+        <filename>postgresql.conf</filename> file or on the server command line.
+       </para>
+       <para>
+        The default value is 10 seconds.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry id="foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+      <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Terminate foreign transaction resolver processes that don't have any foreign
+        transactions to resolve longer than the specified number of milliseconds.
+        A value of zero disables the timeout mechanism.  You should set this value to
+        zero only if you set <varname>max_foreign_transaction_resolvers</varname> as
+        much as databases you have. This parameter can only be set in the
+        <filename>postgresql.conf</filename> file or on the server command line.
+       </para>
+       <para>
+        The default value is 60 seconds.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     </variablelist>
+    </sect2>
+
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -7826,6 +7921,35 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-foreign-transaction">
+    <title>Foreign Transaction Management</title>
+
+    <variablelist>
+
+     <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophase_commit">
+      <term><varname>foreign_twophase_commit</varname> (<type>bool</type>)
+       <indexterm>
+        <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+       </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies whether transaction commit will wait for all involving foreign transaction
+        to be resolved before the command returns a "success" indication to the client.
+        Both <varname>max_prepared_foreign_transactions</varname> and
+        <varname>max_foreign_transaction_resolvers</varname> must be non-zero value to
+        allow foreign twophase commit to be used.
+       </para>
+       <para>
+        This parameter can be changed at any time; the behavior for any one transaction
+        is determined by the setting in effect when it commits.
+       </para>
+      </listitem>
+     </varlistentry>
+    </variablelist>
+
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 4ce88dd..24c635c 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1390,6 +1390,109 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     If an FDW wishes to support <firstterm>atomic commit</firstterm>
+     (as described in <xref linkend="fdw-transaction-managements"/>), it must call the
+     registrasaction function <function>FdwXactRegisterForeignTransaction</function>
+     and provide the following callback functions:
+    </para>
+
+    <para>
+<programlisting>
+bool
+PrepareForeignTransaction(ForeignTransaction *foreign_xact);
+</programlisting>
+    Prepare a foreign transaction identified by <varname>foreign_xact</varname>.
+    This function is called at the pre-commit phase of the local
+    transaction if atomic commit is
+    required. Returning <literal>true</literal> means that preparing
+    the foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(ForeignTransaction *foreign_xact);
+</programlisting>
+    Commit a not-prepared foreign transaction identified by
+    <varname>foreign_xact</varname>.
+    This function is called at the pre-commit phase of local
+    transaction if atomic commit is not required. The atomic
+    commit is not required either when we modified data on
+    only one server including local server or when user doesn't
+    request atomic commit by <xref linkend="guc-foreign-twophase-commit"/>.
+    Returning <literal>true</literal> means that commit the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(ForeignTransaction *foreign_xact);
+</programlisting>
+    Rollback a not-prepared foreign transaction identified by
+    <varname>foreign_xact</varname>.
+    This function is called at the end of local transaction after
+    rollbacked locally either when user requested rollback or when
+    any error occurs within the transaction. This function could
+    be called recursively if any error occurs during rollback the
+    foreign transaction for whatever reason. You need to track
+    recursion and prevent this function from being called infinitely.
+    Returning <literal>true</literal> means that rollback the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+ResolvePreparedForeignTransaction(ForeignTransaction *foreign_xact,
+                                  bool is_commit);
+</programlisting>
+    Commit or rollback the prepared foreign transaction identified
+    by <varname>foreign_xact</varname>. on a connection to foreign server
+    When <varname>is_commit</varname> is true, it indicate that the foreign
+    transaction should be committed.
+    This function normally is called by the foreign transaction resolver
+    process but can also be called by <function>pg_resovle_fdw_xacts</function>
+    function. In the resolver process, this function is called either
+    when a backend requests the resolver process to resolve a distributed
+    transaction after prepared or when a database has dangling
+    transaction. Returning <literal>true</literal> means that resolving
+    the foreign transaction got successful.
+    In abort case, please note that the prepared foreign transaction
+    having identifier <varname>foreign__xact->fx_id</varname> might not
+    exist on the foreign server. If you failed to resolve the foreign
+    transaction due to undefined object error
+    (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) you should regards
+    it as success and return <literal>true</literal>.
+    </para>
+    <para>
+<programlisting>
+bool
+IsTwoPhaseCommitEnabled(Oid serverid);
+</programlisting>
+    Return <literal>true</literal> if foreign server identified by
+    <literal>serverid</literal> is capable of two-phase commit protocol.
+    This function is called when the transaction begins to modify data on
+    the foreign server. Return <literal>false</literal> indicates that
+    the current transaction cannot use atomic commit even if atomic commit
+    is requested by user.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>. To get informations of FDW-related
+      objects, you can use given a <literal>ForeignTransaction</literal>
+      instead (see <filename>foreign/fdwxact.h</filename> for details).
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1835,4 +1938,101 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+    <title>Transaction managements for Foreign Data Wrappers</title>
+
+    <sect2 id="fdw-transaction-atomic-commit">
+     <title>Atomic commit among multiple foreign servers</title>
+
+     <para>
+      <productname>PostgreSQL</productname> foreign transaction manager
+      allows FDWs to read and write data on foreign server within a transaction while
+      maintaining atomicity of the foreign data (aka atomic commit). Using
+      atomic commit, it guarantees that a distributed transaction is committed
+      or rollbacked on all participants foreign
+      server.  To achieve atomic commit, <productname>PostgreSQL</productname>
+      employees two-phase commit protocol, which is a type of atomic commitment
+      protocol. Every FDW that wish to support atomic commit
+      is required to support transaction management callback routines
+      (see <xref linkend="fdw-callbacks-transaction-managements"/> for details)
+      and register the foreign transaction using
+      <function>FdwXactRegisterForeignTransaction</function> when starting a
+      transaction on the foreign server. Transaction of registered foreign server
+      is managed by the foreign transaction manager.
+<programlisting>
+void
+FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, char *fx_id)
+</programlisting>
+    This function should be called when a transaction starts on the foreign server.
+    <varname>serverid</varname> and <varname>userid</varname> are <type>OID</type>s
+    which specify the transaction starts on what server by who. <varname>fx_id</varname>
+    is null-terminated string which is an identifer of foreign transaction and it
+    will be passed when transaction management APIs is called. The length of
+    <varname>fx_id</varname> must be less than 200 bytes. Also this identifier
+    must be unique enough so that it doesn't conflict other concurrent foreign
+    transactions. <varname>fx_id</varname> can be <literal>NULL</literal>.
+    If it's <literal>NULL</literal>, a transaction identifier is automacitally
+    generated with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    Since this identifier is used per foreign transaction and the xid of unresolved
+    distributed transaction never reused, an auto-generated identifier is fairly
+    enough to ensure uniqueness. It's recommended to generate foreign transaction
+    identifier in FDW if the format of auto-generated identifier doesn't match
+    the requirement of the foreign server.
+    </para>
+
+     <para>
+      An example of such transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+     </para>
+
+     <para>
+      When a transaction starts on the foreign server, FDW that wishes atomic
+      commit must register the foreign transaction as a participant by calling
+      <function>FdwXactRegisterForeignTransaction</function>. Also during
+      transaction, <function>IsTwoPhaseCommitEnabled</function> is called whenever
+      the transaction begins to modify data on the foreign server. If FDW wishes
+      atomic commit <function>IsTwoPhaseCommitEnabled</function> must return
+      <literal>true</literal>. All foreign transaction participants must
+      return <literal>true</literal> to achieve atomic commit.
+     </para>
+
+     <para>
+      During pre-commit phase of local transaction, the foreign transaction manager
+      persists the foreign transaction information to the disk and WAL, and then
+      prepare all foreign transaction by calling <function>PrepareForeignTransaction</function>
+      if two-phase commit protocol is required. Two-phase commit is required only if
+      the transaction modified data on more than one servers including the local
+      server and user requests atomic commit. <productname>PostgreSQL</productname>
+      can commit locally and go to the next step if and only if all preparing foreign
+      transactions got successful. If two-phase commit is not required, the foreign
+      transaction manager commits a transaction on the foreign server by calling
+      <function>CommitForeignTransaction</function> and then
+      <productname>PostgreSQL</productname> commits locally. The foreign transaction
+      manager doesn't do any further change on foreign transactions from this point
+      forward. If any failure happens for whatever reason, for example a network
+      failure or user request until <productname>PostgreSQL</productname> commits
+      locally the foreign transaction manager changes over to rollback and calls
+      <function>RollbackForeignTransaction</function> for every foreign servers to
+      close the current transaction on foreign servers.
+     </para>
+
+     <para>
+      When two-phase commit is required, after committed locally, each the transaction
+      commits will wait for all prepared foreign transaction to be resolved before
+      the commit completes. The foreign transaction resolver is responsible for
+      foreign transaction resolution. <function>ResolverForeignTransaction</function>
+      is called by the foreign transaction resolver process when it resolves a foreign
+      transactions. <function>ResolveForeignTransaction</function> is also be called
+      when user execute <function>pg_resovle_fdw_xact</function> function.
+     </para>
+    </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 9a7f683..8ed007c 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -20755,6 +20755,57 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-fdw-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_fdw_xacts</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_fdw_xacts</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function search for foreign transaction
+        matching the arguments and resolves then. This function won't resolve
+        a foreign transaction which is in progress, or one that is locked by some
+        other backend.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_fdw_xact</function>
+        except it remove foreign transaction entry without resolving.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 0484cfa..635a5e7 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -332,6 +332,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_fdw_xact_resolver</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-resolver-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1194,6 +1202,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
          <entry><literal>LogicalLauncherMain</literal></entry>
          <entry>Waiting in main loop of logical launcher process.</entry>
         </row>
@@ -1405,6 +1421,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
+        <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
          <entry>Waiting during base backup when throttling activity.</entry>
@@ -2214,6 +2234,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-resolver-view" xreflabel="pg_stat_fdw_xact_resolver">
+   <title><structname>pg_stat_fdw_xact_resolver</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index 5514db1..742e825 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -8,9 +8,9 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
-	   gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
-	   mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
-	   smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdwxactdesc.o \
+	genericdesc.o  gindesc.o gistdesc.o hashdesc.o heapdesc.o \
+	logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \
+	seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000..3705104
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,65 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdw_xactdesc.c
+ *		PostgreSQL distributed transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/backend/access/transam/fdw_xactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "foreign/fdwxact_xlog.h"
+
+void
+fdw_xact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		FdwXactOnDiskData *fdw_insert_xlog = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_insert_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_insert_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_insert_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_insert_xlog->local_xid);
+		/* TODO: This should be really interpreted by each FDW */
+
+		/*
+		 * TODO: we also need to assess whether we want to add this
+		 * information
+		 */
+		appendStringInfo(buf, " foreign transaction info: %s",
+						 fdw_insert_xlog->fdw_xact_id);
+	}
+	else
+	{
+		xl_fdw_xact_remove *fdw_remove_xlog = (xl_fdw_xact_remove *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_remove_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_remove_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_remove_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_remove_xlog->xid);
+	}
+
+}
+
+const char *
+fdw_xact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDW_XACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDW_XACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 00741c7..023a7c5 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -112,14 +112,16 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_prepared_xacts=%d max_locks_per_xact=%d "
 						 "wal_level=%s wal_log_hints=%s "
-						 "track_commit_timestamp=%s",
+						 "track_commit_timestamp=%s "
+						 "max_prepared_foreign_xacts=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_prepared_xacts,
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47..f15c83a 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -12,9 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
-	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
-	xact.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o generic_xlog.o multixact.o \
+	parallel.o rmgr.o slru.o subtrans.o timeline.o transam.o twophase.o \
+	twophase_rmgr.o varsup.o xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9368b56..b5c3502 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -24,6 +24,7 @@
 #include "commands/dbcommands_xlog.h"
 #include "commands/sequence.h"
 #include "commands/tablespace.h"
+#include "foreign/fdwxact.h"
 #include "replication/message.h"
 #include "replication/origin.h"
 #include "storage/standby.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 3942734..839e768 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -89,6 +89,7 @@
 #include "access/xlogreader.h"
 #include "catalog/pg_type.h"
 #include "catalog/storage.h"
+#include "foreign/fdwxact.h"
 #include "funcapi.h"
 #include "miscadmin.h"
 #include "pg_trace.h"
@@ -844,6 +845,35 @@ TwoPhaseGetGXact(TransactionId xid)
 }
 
 /*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
+/*
  * TwoPhaseGetDummyProc
  *		Get the dummy backend ID for prepared transaction specified by XID
  *
@@ -2316,6 +2346,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2375,6 +2411,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 875be18..c4c879d 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -36,6 +36,7 @@
 #include "commands/tablecmds.h"
 #include "commands/trigger.h"
 #include "executor/spi.h"
+#include "foreign/fdwxact.h"
 #include "libpq/be-fsstubs.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -1108,6 +1109,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_twophase_for_ac;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1116,6 +1118,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_twophase_for_ac = ForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1154,12 +1157,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_twophase_for_ac)
 			goto cleanup;
 	}
 	else
@@ -1317,6 +1321,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_twophase_for_ac && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -1955,6 +1967,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2110,6 +2125,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2197,6 +2213,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2385,6 +2403,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2589,6 +2608,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7375a78..2a168cd 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -40,6 +40,7 @@
 #include "catalog/pg_control.h"
 #include "catalog/pg_database.h"
 #include "commands/tablespace.h"
+#include "foreign/fdwxact.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "port/atomics.h"
@@ -5267,6 +5268,7 @@ BootStrapXLOG(void)
 	ControlFile->MaxConnections = MaxConnections;
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6354,6 +6356,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -6878,14 +6883,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdw_xact, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -7077,7 +7083,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7583,6 +7592,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7901,6 +7911,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9217,6 +9230,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9650,7 +9664,8 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9682,6 +9697,7 @@ XLogReportParameters(void)
 		ControlFile->MaxConnections = MaxConnections;
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9887,6 +9903,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -10085,6 +10102,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->MaxConnections = xlrec.MaxConnections;
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 7251552..5fa6065 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -291,6 +291,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_prepared_fdw_xacts AS
+       SELECT * FROM pg_prepared_fdw_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
 	l.objoid, l.classoid, l.objsubid,
@@ -773,6 +776,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_fdwxact_resolvers AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_fdwxact_resolver() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index e5dd995..50c31ef 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -28,6 +28,7 @@
 #include "catalog/pg_user_mapping.h"
 #include "commands/defrem.h"
 #include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
 #include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "parser/parse_func.h"
@@ -1093,6 +1094,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1407,6 +1420,16 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
 	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srv->serverid,
+						useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
+	/*
 	 * Do the deletion
 	 */
 	object.classId = UserMappingRelationId;
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index ec7a526..ea31749 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -19,6 +19,7 @@
 #include "executor/execPartition.h"
 #include "executor/executor.h"
 #include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "nodes/makefuncs.h"
@@ -744,7 +745,10 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+		FdwXactMarkForeignTransactionModified(partRelInfo, 0);
+	}
 
 	MemoryContextSwitchTo(oldContext);
 
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index a2a28b7..30a0b66 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,9 +22,11 @@
  */
 #include "postgres.h"
 
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
 #include "utils/memutils.h"
 #include "utils/rel.h"
 
@@ -224,7 +226,13 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+
+		/* Mark this transaction modified data on the foreign server */
+		FdwXactMarkForeignTransactionModified(estate->es_result_relation_info,
+										 eflags);
+	}
 	else
 		fdwroutine->BeginForeignScan(scanstate, eflags);
 
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index bf0d5e8..283bfaf 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -44,6 +44,8 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "storage/bufmgr.h"
@@ -2317,6 +2319,9 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 															 fdw_private,
 															 i,
 															 eflags);
+
+			/* Mark this transaction modified data on the foreign server */
+			FdwXactMarkForeignTransactionModified(resultRelInfo, eflags);
 		}
 
 		resultRelInfo++;
diff --git a/src/backend/foreign/Makefile b/src/backend/foreign/Makefile
index 85aa857..4329d3e 100644
--- a/src/backend/foreign/Makefile
+++ b/src/backend/foreign/Makefile
@@ -12,6 +12,6 @@ subdir = src/backend/foreign
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS= foreign.o
+OBJS= foreign.o fdwxact.o fdwxact_launcher.o fdwxact_resolver.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/foreign/fdwxact.c b/src/backend/foreign/fdwxact.c
new file mode 100755
index 0000000..d284861
--- /dev/null
+++ b/src/backend/foreign/fdwxact.c
@@ -0,0 +1,2762 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL distributed transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers atomically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * When a foreign data wrapper starts transaction on a foreign server
+ * that is capable of two-phase commit protocol, it's required to register
+ * the foreign transaction using function FdwXactRegisterTransaction() in order
+ * to participate to a group for atomic commit. Participants are identified
+ * by oid of foreign server and user. When the foreign transaction begins
+ * to modify data it's required to mark it as modified using
+ * FdwXactMarkForeignTransactionModified()
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. After committing or rolling back locally, we
+ * notify the resolver process and tell it to commit or roll back those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish,
+ * and so that we're not trying to locally do work that might fail when an
+ * ERROR after already committed.
+ *
+ * Two-phase commit protocol is required if the transaction modified
+ * two or more servers including itself. In other case, all foreign transactions
+ * are committed during pre-commit.
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. dangling
+ * transaction). Dangling transactions are processed by the resolve process
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * 	* On PREPARE redo we add the foreign transaction to FdwXactCtl->fdw_xacts.
+ *	  We set fdw_xact->inredo to true for such entries.
+ *	* On Checkpoint redo, we iterate through FdwXactCtl->fdw_xacts entries that
+ *	  have set fdw_xact->inredo true and are behind the redo_horizon. We save
+ *    them to disk and then set fdw_xact->ondisk to true.
+ *	* On COMMIT and ABORT we delete the entry from FdwXactCtl->fdw_xacts.
+ *	  If fdw_xact->ondisk is true, we delete the corresponding file from
+ *	  the disk as well.
+ *  * RecoverFdwXacts loads all foreign transaction entries from disk into
+ *    memory at server startup.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/foreign/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/fdwxact.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/fdwxact_launcher.h"
+#include "foreign/fdwxact_xlog.h"
+#include "foreign/resolver_internal.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Is atomic commit requested by user? */
+#define AtomicCommitRequested() \
+	(foreign_twophase_commit == true && \
+	 max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Structure to bundle the foreign transaction participant */
+typedef struct FdwXactParticipant
+{
+	Oid			serverid;
+	Oid			userid;
+
+	/*
+	 * Pointer to a FdwXact entry in global entry. NULL if
+	 * this foreign transaction is registered but not inserted
+	 * yet.
+	 */
+	FdwXact		fdw_xact;
+	char		*fdw_xact_id;
+
+	/* true if this transaction modified data on the foreign server */
+	bool		modified;
+
+	/*
+	 * This is initialized at foreign transaction registration and
+	 * passed to API functions.
+	 */
+	ForeignTransaction foreign_xact;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact;
+	CommitForeignTransaction_function	commit_foreign_xact;
+	RollbackForeignTransaction_function	rollback_foreign_xact;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit.
+ * This list has only foreign servers that are capable of two-phase
+ * commit protocol.
+ */
+List *FdwXactParticipantsForAC = NIL;
+
+/*
+ * This struct tracks all participants involved with transaction 'xid'.
+ */
+typedef struct FdwXactStateCacheEntry
+{
+	/* Key -- must be first */
+	TransactionId	xid;
+
+	/* List of FdwXacts involved with the xid */
+	FdwXact	participants;
+} FdwXactStateCacheEntry;
+static HTAB	*FdwXactStateCache;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDW_XACTS_DIR "pg_fdw_xact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDW_XACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDW_XACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+static FdwXact FdwXactRegisterFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static bool FdwXactResolveForeignTransaction(FdwXact fdw_xact);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactQueueInsert(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid, bool give_warnings);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							  bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid, Oid serverid,
+								  Oid userid, XLogRecPtr insert_start_lsn,
+								  bool give_warnings);
+static List *get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						   bool need_lock);
+static FdwXact get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								bool need_lock);
+static FdwXact get_all_fdw_xacts(int *length);
+static FdwXact insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							   char *fdw_xact_id);
+static char *generate_fdw_xact_identifier(Oid serverid, Oid userid);
+static void remove_fdw_xact(FdwXact fdw_xact);
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+bool foreign_twophase_commit = false;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * This function aimes to be called by FDW when foreign transaction
+ * starts. The foreign server identified by given server id must
+ * support atomic commit APIs. The foreign transaction is identified
+ * by given identifier 'fdw_xact_id' which can be NULL. If it's NULL,
+ * we construct an unique identifer.
+ *
+ * After registered, foreign transaction of participants are managed
+ * by foreign transaction manager until the end of the distributed
+ * transaction.
+ */
+void
+FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, char *fx_id)
+{
+	FdwXactParticipant	*fdw_part;
+	ListCell   			*lc;
+	ForeignServer 		*foreign_server;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	FdwRoutine			*fdw_routine;
+	MemoryContext		old_context;
+
+	/* Check length of foreign transaction identifier */
+	if (fx_id != NULL && strlen(fx_id) >= NAMEDATALEN)
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifer \"%s\" is too long",
+						fx_id),
+				 errdetail("foreign transaction identifier must be less than %d characters.",
+						   NAMEDATALEN)));
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_xact_resolvers to a nonzero value.")));
+
+	/* Duplication check */
+	foreach(lc, FdwXactParticipantsForAC)
+	{
+		fdw_part = lfirst(lc);
+
+		/* Quick return if there is already registered connection */
+		if (fdw_part->serverid == serverid && fdw_part->userid == userid)
+			ereport(ERROR,
+					(errmsg("attempt to start transction again on server %u user %u",
+							serverid, userid)));
+	}
+
+	/*
+	 * Participants information is needed at the end of a transaction, when
+	 * system cache are not available. so save it in TopTransactionContext
+	 * before hand so that these can live until the end of transaction.
+	 */
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	foreign_server = GetForeignServer(serverid);
+	fdw = GetForeignDataWrapper(foreign_server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+	user_mapping = GetUserMapping(userid, serverid);
+
+	/* Make sure that the FDW has transaction handlers */
+	if (!fdw_routine->PrepareForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function provided for preparing foreign transaction for FDW %s",
+						fdw->fdwname)));
+	if (!fdw_routine->CommitForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to commit a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+	if (!fdw_routine->RollbackForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to rollback a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+
+	/* Generate foreign transaction identifier if not provided */
+	if (fx_id ==  NULL)
+		fx_id = generate_fdw_xact_identifier(serverid, userid);
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->serverid = serverid;
+	fdw_part->userid = userid;
+	fdw_part->fdw_xact_id = fx_id;
+	fdw_part->fdw_xact = NULL;
+	fdw_part->modified = false;	/* by default */
+	fdw_part->foreign_xact.server = foreign_server;
+	fdw_part->foreign_xact.usermapping = user_mapping;
+	fdw_part->foreign_xact.fx_id = fx_id;
+	fdw_part->prepare_foreign_xact = fdw_routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact = fdw_routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact = fdw_routine->RollbackForeignTransaction;
+
+	/* Add this foreign connection to the participants list */
+	FdwXactParticipantsForAC = lappend(FdwXactParticipantsForAC, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_context);
+
+	return;
+}
+
+/*
+ * Remember the registered foreign transaction modified data . This function
+ * is called when the executor begins to modify data on a foreign server
+ * regardless the foreign server is capable of two-phase commit protocol.
+ * Marking it will be used to determine we must use two-phase commit protocol
+ * at commit. This function also checks if the begin modified foreign server
+ * is capable of two-phase commit or not. If it doesn't support, we remember
+ * it.
+ */
+void
+FdwXactMarkForeignTransactionModified(ResultRelInfo *resultRelInfo, int flags)
+{
+	Relation			rel = resultRelInfo->ri_RelationDesc;
+	FdwXactParticipant	*fdw_part;
+	ForeignTable		*ftable;
+	ListCell   			*lc;
+	Oid					userid;
+	Oid					serverid;
+
+	bool found = false;
+
+	/* Quick return if user not request */
+	if (!AtomicCommitRequested())
+		return;
+
+	/* Do nothing in EXPLAIN (no ANALYZE) case */
+	if (flags && EXEC_FLAG_EXPLAIN_ONLY)
+		return;
+
+	ftable = GetForeignTable(RelationGetRelid(rel));
+
+	/*
+	 * If the being modified foreign server doesn't or cannot enable
+	 * two-phase commit protocol, mark that we've written such server
+	 * and return.
+	 */
+	if (resultRelInfo->ri_FdwRoutine->IsTwoPhaseCommitEnabled == NULL ||
+		!resultRelInfo->ri_FdwRoutine->IsTwoPhaseCommitEnabled(ftable->serverid))
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		return;
+	}
+
+	/*
+	 * The foreign server being modified supports two-phase commit protocol,
+	 * remember that the foreign transaction modified data.
+	 */
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	serverid = ftable->serverid;
+	foreach(lc, FdwXactParticipantsForAC)
+	{
+		fdw_part = lfirst(lc);
+
+		if (fdw_part->serverid == serverid && fdw_part->userid == userid)
+		{
+			fdw_part->modified = true;
+			found = true;
+			break;
+		}
+	}
+
+	if (!found)
+		elog(ERROR, "attempt to mark unregistered foreign server %u, user %u as modified",
+			 serverid, userid);
+}
+
+/*
+ * FdwXactShmemSize
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdw_xacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	/* Size for shared cache entry */
+	size = MAXALIGN(size);
+	size = add_size(size, hash_estimate_size(max_prepared_foreign_xacts,
+											 sizeof(FdwXactStateCacheEntry)));
+
+	return size;
+}
+
+/*
+ * FdwXactShmemInit
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of
+ * FdwXactCtlData structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdw_xacts;
+		HASHCTL		info;
+		long		max_hash_size;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->freeFdwXacts = NULL;
+		FdwXactCtl->numFdwXacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdw_xacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdw_xacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdw_xacts[cnt].status = FDW_XACT_INITIAL;
+			fdw_xacts[cnt].fxact_free_next = FdwXactCtl->freeFdwXacts;
+			FdwXactCtl->freeFdwXacts = &fdw_xacts[cnt];
+		}
+
+		/* Initialize shared state cache hash table */
+		MemSet(&info, 0, sizeof(info));
+		info.keysize = sizeof(TransactionId);
+		info.entrysize = sizeof(FdwXactStateCacheEntry);
+		max_hash_size = max_prepared_foreign_xacts;
+
+		FdwXactStateCache = ShmemInitHash("FdwXact hash",
+										  max_hash_size,
+										  max_hash_size,
+										  &info,
+										  HASH_ELEM | HASH_BLOBS);
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * PreCommit_FdwXacts
+ *
+ * This function prepares all foreign transaction participants if atomic commit
+ * is required. Otherwise commits them without preparing.
+ *
+ * If atomic commit is requested by user (that is, foreign_twophase_commit is on),
+ * every participants must enable two-phase commit. If we manage all foreign
+ * transactions involving with a transaction we can commit foreign transactions
+ * on foreign server that doesn't use two-phase commit here and commit others
+ * at post-commit phase, but we don't do that. Because (1) it doesn't satisfy
+ * the atomic commit semantics at all and (2) it requires all FDWs to register
+ * foreign server anyway, which breaks backward compatibility.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipantsForAC == NIL)
+		return;
+
+	/*
+	 * If user requires the atomic commit semantics, we don't allow COMMIT if we've
+	 * modified data on  foreign servers both that can execute two-phase commit
+	 * protocol and that cannot.
+	 */
+	if (foreign_twophase_commit == true && MyXactFlags & XACT_FLAGS_FDWNOPREPARE)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	if (ForeignTwophaseCommitRequired())
+	{
+		/* Prepare the transactions on the all foreign servers */
+		FdwXactPrepareForeignTransactions();
+	}
+	else
+	{
+		ListCell   *lc;
+
+		Assert(list_length(FdwXactParticipantsForAC) == 1);
+
+		/* Two-phase commit is not required, commit them one by one */
+		foreach(lc, FdwXactParticipantsForAC)
+		{
+			FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+			/* Commit foreign transaction */
+			if (!fdw_part->commit_foreign_xact(&fdw_part->foreign_xact))
+				ereport(ERROR,
+						(errmsg("could not commit foreign transaction on server %s",
+								fdw_part->foreign_xact.server->servername)));
+		}
+
+		/* Forget all participants */
+		FdwXactParticipantsForAC = NIL;
+	}
+}
+
+/*
+ * FdwXactPrepareForeignTransactions
+ *
+ * Prepare all foreign transaction participants.  This function creates a prepared
+ * participants chain whenever we prepared a foreign transaction. The prepared
+ * participants chain is used to access all participants of distributed transaction
+ * quickly. If any one of them fails to prepare or raises an error, we change over
+ * to aborts.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell   *lcell;
+	FdwXact		prev_fxact = NULL;
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactParticipantsForAC)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXact		fxact;
+
+		/*
+		 * Register the foreign transaction entry. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fxact = FdwXactRegisterFdwXactEntry(GetTopTransactionId(), fdw_part);
+
+		/*
+		 * Between FdwXactRegisterFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal). During abort processing,
+		 * we might try to resolve a never-prepared transaction, and get an error.
+		 * This is fine as long as the FDW provides us unique prepared transaction
+		 * identifiers.
+		 */
+		if (!fdw_part->prepare_foreign_xact(&fdw_part->foreign_xact))
+		{
+			/* Failed to prepare, change over aborts */
+			ereport(ERROR,
+					(errmsg("could not prepare transaction on foreign server %s",
+							fdw_part->foreign_xact.server->servername)));
+		}
+
+		/* Preparation is success, update its status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdw_part->fdw_xact->status = FDW_XACT_PREPARED;
+		fdw_part->fdw_xact = fxact;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Create a prepared participants chain, which is link-ed FdwXact entries
+		 * involving with this transaction. The head entry is remembered in hash
+		 * table and subsequent entries is liked from the previous entry.
+		 */
+		if (!prev_fxact)
+		{
+			FdwXactStateCacheEntry	*fxact_entry;
+			bool				found;
+
+			LWLockAcquire(FdwXactLock,LW_EXCLUSIVE);
+			fxact_entry = (FdwXactStateCacheEntry *) hash_search(FdwXactStateCache,
+																 (void *) &(fxact->local_xid),
+																 HASH_ENTER, &found);
+			LWLockRelease(FdwXactLock);
+			Assert(!found);
+
+			/* Set the first participant */
+			fxact_entry->participants = fxact;
+		}
+		else
+		{
+			/* Append others to the tail */
+			Assert(fxact->fxact_next == NULL);
+			prev_fxact->fxact_next = fxact;
+		}
+
+		prev_fxact = fxact;
+	}
+}
+
+/*
+ * FdwXactRegisterFdwXactEntry
+ *
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and will
+ * be persisted to the disk under pg_fdw_xact directory when checkpoint.
+ */
+static FdwXact
+FdwXactRegisterFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fxact;
+	FdwXactOnDiskData	*fxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact = insert_fdw_xact(MyDatabaseId, xid, fdw_part->serverid,
+							fdw_part->userid, fdw_part->fdw_xact_id);
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->registered_backend = MyBackendId;
+	fdw_part->fdw_xact = fxact;
+	LWLockRelease(FdwXactLock);
+
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdw_xact_id);
+	data_len = data_len + strlen(fdw_part->fdw_xact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fxact_file_data->dbid = MyDatabaseId;
+	fxact_file_data->local_xid = xid;
+	fxact_file_data->serverid = fdw_part->serverid;
+	fxact_file_data->userid = fdw_part->userid;
+	memcpy(fxact_file_data->fdw_xact_id, fdw_part->fdw_xact_id,
+		   strlen(fdw_part->fdw_xact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fxact_file_data, data_len);
+	fxact->insert_end_lsn = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_INSERT);
+	XLogFlush(fxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact->valid = true;
+	LWLockRelease(FdwXactLock);
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fxact_file_data);
+	return fxact;
+}
+
+/*
+ * insert_fdw_xact
+ *
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				char *fdw_xact_id)
+{
+	int i;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		fxact = FdwXactCtl->fdw_xacts[i];
+		if (fxact->dbid == dbid &&
+			fxact->local_xid == xid &&
+			fxact->serverid == serverid &&
+			fxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->freeFdwXacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fxact = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fxact->fxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->numFdwXacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts++] = fxact;
+
+	fxact->registered_backend = InvalidBackendId;
+	fxact->dbid = dbid;
+	fxact->local_xid = xid;
+	fxact->serverid = serverid;
+	fxact->userid = userid;
+	fxact->insert_start_lsn = InvalidXLogRecPtr;
+	fxact->insert_end_lsn = InvalidXLogRecPtr;
+	fxact->valid = false;
+	fxact->ondisk = false;
+	fxact->inredo = false;
+	memcpy(fxact->fdw_xact_id, fdw_xact_id, strlen(fdw_xact_id) + 1);
+
+	return fxact;
+}
+
+/*
+ * remove_fdw_xact
+ *
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdw_xact(FdwXact fdw_xact)
+{
+	int			cnt;
+
+	Assert(fdw_xact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Search the slot where this entry resided */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		if (FdwXactCtl->fdw_xacts[cnt] == fdw_xact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (cnt >= FdwXactCtl->numFdwXacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdw_xact->local_xid, fdw_xact->serverid, fdw_xact->userid)));
+
+	/* Remove the entry from active array */
+	FdwXactCtl->numFdwXacts--;
+	FdwXactCtl->fdw_xacts[cnt] = FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts];
+
+	/* Put it back into free list */
+	fdw_xact->fxact_free_next = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fdw_xact;
+
+	/* Reset informations */
+	fdw_xact->status = FDW_XACT_INITIAL;
+	fdw_xact->registered_backend = InvalidBackendId;
+	fdw_xact->fxact_next = NULL;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdw_xact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdw_xact->serverid;
+		record.dbid = fdw_xact->dbid;
+		record.xid = fdw_xact->local_xid;
+		record.userid = fdw_xact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdw_xact_remove));
+		recptr = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true if the current transaction requires foreign two-phase commit
+ * to achieve atomic commit. Foreign two-phase commit is required if we
+ * satisfy either case: we modified data on two or more foreign server, or
+ * we modified both non-temporary relation on local and data on more than
+ * one foreign server.
+ */
+bool
+ForeignTwophaseCommitRequired(void)
+{
+	int	nserverswritten = list_length(FdwXactParticipantsForAC);
+	ListCell*	lc;
+	bool		modified = false;
+
+	/* Return if not requested */
+	if (!AtomicCommitRequested())
+		return false;
+
+	/* Check if we modified data on any foreign server */
+	foreach(lc, FdwXactParticipantsForAC)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+		{
+			modified = true;
+			break;
+		}
+	}
+
+	/* We didn't modify data on any foreign server */
+	if (!modified)
+		return false;
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	return nserverswritten > 1;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * ForgetAllFdwXactParticipants
+ *
+ * Reset all the foreign transaction entries that this backend registered.
+ * If the foreign transaction has the corresponding FdwXact entry, resetting
+ * the registered_backend field means to leave that entry in unresolved state.
+ * If we leaves any entries, we update the oldest xmin of unresolved transaction
+ * so that transaction status of dangling transaction are not truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_left = 0;
+
+	if (FdwXactParticipantsForAC == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(cell, FdwXactParticipantsForAC)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+
+		/* Skip if didn't register FdwXact entry yet */
+		if (fdw_part->fdw_xact == NULL)
+			continue;
+
+		/*
+		 * There is a race condition; the entries of FdwXactParticipantsForAC
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget them. So we need to check if the entries are
+		 * still associated with the transaction.
+		 */
+		if (fdw_part->fdw_xact->registered_backend == MyBackendId)
+		{
+			fdw_part->fdw_xact->registered_backend = InvalidBackendId;
+			n_left++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Update the oldest local transaction of unresolved distributed
+	 * transaction if we leaved any FdwXact entries.
+	 */
+	if (n_left > 0)
+		FdwXactComputeRequiredXmin();
+
+	FdwXactParticipantsForAC = NIL;
+}
+
+/*
+ * AtProcExit_FdwXact
+ *
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+/*
+ * Wait for foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDW_XACT_NOT_WAITING and then change
+ * that state to FDW_XACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransactions a fdwxact resolver changes the
+ * state to FDW_XACT_WAIT_COMPLETE once foreign transactions are resolved.
+ * This backend then resets its state to FDW_XACT_NOT_WAITING.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+	ListCell	*lc;
+	List		*fdwxact_participants = NIL;
+
+	/* Quick exit if atomic commit is not requested */
+	if (!AtomicCommitRequested())
+		return;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDW_XACT_NOT_WAITING);
+
+	if (FdwXactParticipantsForAC != NIL)
+	{
+		/*
+		 * If we're waiting for foreign transactions to be resolved that
+		 * we've prepared just before, use the participants list.
+		 */
+		Assert(MyPgXact->xid == wait_xid);
+		fdwxact_participants = FdwXactParticipantsForAC;
+	}
+	else
+	{
+		FdwXactStateCacheEntry *fdwxact_entry;
+		bool found;
+
+		/*
+		 * If we're waiting for foreign transactions to be resolved that
+		 * is part of a local prepared transaction that is mark as
+		 * prepared during running, since these entries exist in the hash
+		 * table we construct the participants list from the entry.
+		 */
+		Assert(FdwXactStateCache);
+		fdwxact_entry = (FdwXactStateCacheEntry *) hash_search(FdwXactStateCache,
+															   (void *) &wait_xid,
+															   HASH_FIND, &found);
+
+		if (found)
+		{
+			FdwXact	fdwxact;
+
+			for (fdwxact = fdwxact_entry->participants;
+				 fdwxact != NULL;
+				 fdwxact = fdwxact->fxact_next)
+				fdwxact_participants = lappend(fdwxact_participants, fdwxact);
+		}
+	}
+
+	/*
+	 * Otherwise, construct the participants list by scanning the global
+	 * array. This can happen in the case where we restarts after PREPARE'd
+	 * a distributed transaction and then are trying to resolve it.
+	 */
+	if (fdwxact_participants == NIL)
+		fdwxact_participants = get_fdw_xacts(MyDatabaseId, wait_xid,
+											 InvalidOid, InvalidOid, true);
+
+	/* Exit if we found no foreign transaction to resolve */
+	if (fdwxact_participants == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(lc, fdwxact_participants)
+	{
+		FdwXact fdw_xact = (FdwXact) lfirst(lc);
+
+		/* Don't overwrite status if fate has been determined */
+		if (fdw_xact->status == FDW_XACT_PREPARED)
+			fdw_xact->status = (is_commit ?
+								FDW_XACT_COMMITTING_PREPARED :
+								FDW_XACT_ABORTING_PREPARED);
+	}
+
+	/* Set backend status and enqueue itself */
+	MyProc->fdwXactState = FDW_XACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	FdwXactQueueInsert();
+	LWLockRelease(FdwXactLock);
+
+	/* Launch a resolver process if not yet, or wake it up */
+	fdwxact_maybe_launch_resolver(false);
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDW_XACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDW_XACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDW_XACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+
+	/*
+	 * Forget the list of locked entries, also means that the entries
+	 * that could not resolved are remained as dangling transactions.
+	 */
+	ForgetAllFdwXactParticipants();
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Acquire FdwXactLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Insert MyProc into the tail of FdwXactQueue.
+ */
+static void
+FdwXactQueueInsert(void)
+{
+	SHMQueueInsertBefore(&(FdwXactRslvCtl->FdwXactQueue),
+						 &(MyProc->fdwXactLinks));
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * Create and initialize an FdwXactResolveState which is used
+ * for resolution of foreign transactions.
+ */
+FdwXactResolveState *
+CreateFdwXactResolveState(void)
+{
+	FdwXactResolveState *frstate = palloc0(sizeof(FdwXactResolveState));
+
+	frstate->dbid = MyDatabaseId;
+	frstate->fdwxact = NULL;
+	frstate->waiter = NULL;
+
+	return frstate;
+}
+
+/*
+ * Resolve one distributed transaction. The target distributed transaction
+ * is fetched from shmem queue and its participants are fetched from either
+ * shmem hash table or global array. Release the waiter and return true only
+ * if we resolved the all of the foreign transaction participants. Return
+ * false if we flied to resolve any of them.
+ *
+ * To ensure the order of registered distributed transaction to the queue, we
+ * must not go the next distributed transaction until all of participants are
+ * resolved. The failed foreign transactions will be retried at the next execution.
+ */
+bool
+FdwXactResolveDistributedTransaction(FdwXactResolveState *frstate)
+{
+	FdwXactStateCacheEntry	*fdwxact_entry = NULL;
+	volatile FdwXact	fdwxacts_failed_to_resolve = NULL;
+	bool				all_resolved = false;
+
+	Assert(frstate->dbid == MyDatabaseId);
+
+	/* Get a new waiter, if not exists */
+	if (frstate->waiter == NULL)
+	{
+		PGPROC	*proc;
+
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+		/* Fetch a waiter from beginning of the queue */
+		while ((proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->FdwXactQueue),
+											   &(FdwXactRslvCtl->FdwXactQueue),
+											   offsetof(PGPROC, fdwXactLinks))) != NULL)
+		{
+			/* Found a waiter */
+			if (proc->databaseId == frstate->dbid)
+				break;
+		}
+
+		LWLockRelease(FdwXactLock);
+
+		/* If no waiter, there is no job */
+		if (!proc)
+			return false;
+
+		Assert(TransactionIdIsValid(proc->fdwXactWaitXid));
+		frstate->waiter = proc;
+	}
+
+	/* Get foreign transaction participants */
+	if (frstate->fdwxact == NULL)
+	{
+		bool found;
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+		/* Search FdwXact entries from the hash table by the local transaction id */
+		fdwxact_entry =
+			(FdwXactStateCacheEntry *) hash_search(FdwXactStateCache,
+												   (void *) &(frstate->waiter->fdwXactWaitXid),
+												   HASH_FIND, &found);
+
+		if (found)
+			frstate->fdwxact = fdwxact_entry->participants;
+		else
+		{
+			int i;
+			FdwXact entries_to_resolve = NULL;
+			FdwXact prev_fx = NULL;
+
+			/*
+			 * The fdwxact entry doesn't exist in the hash table in case where
+			 * a prepared transaction is resolved after recovery. In this case,
+			 * we construct a list of fdw xact entries by scanning over the
+			 * FdwXactCtl->fdw_xacts list.
+			 */
+			for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+			{
+				FdwXact fdw_xact = FdwXactCtl->fdw_xacts[i];
+
+				if (fdw_xact->dbid == frstate->dbid &&
+					fdw_xact->local_xid == frstate->waiter->fdwXactWaitXid)
+				{
+					if (!entries_to_resolve)
+						entries_to_resolve = fdw_xact;
+
+					/* Link from previous entry to this entry */
+					if (prev_fx)
+						prev_fx->fxact_next = fdw_xact;
+
+					prev_fx = fdw_xact;
+				}
+			}
+
+			frstate->fdwxact = entries_to_resolve;
+		}
+
+		LWLockRelease(FdwXactLock);
+	}
+
+	Assert(frstate->fdwxact != NULL);
+
+	/* Resolve all foreign transactions one by one */
+	while (frstate->fdwxact != NULL)
+	{
+		volatile FdwXact cur_fdwxact = frstate->fdwxact;
+		volatile FdwXact fdwxact_next = NULL;
+
+		/*
+		 * Remember the next FdwXact entry to resolve as the current entry will
+		 * be removed after resolved from the list.
+		 */
+		fdwxact_next = cur_fdwxact->fxact_next;
+
+		/* Resolve a foreign transaction */
+		if (!FdwXactResolveForeignTransaction(cur_fdwxact))
+		{
+			ForeignServer *fserver;
+
+			CHECK_FOR_INTERRUPTS();
+
+			/* Failed to resolve. Remember it for the next execution */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			if (fdwxacts_failed_to_resolve == NULL)
+			{
+				/*
+				 * For the first failed entry, reset its next pointer
+				 * and append it to the head of list.
+				 */
+				cur_fdwxact->fxact_next = NULL;
+				fdwxacts_failed_to_resolve = cur_fdwxact;
+			}
+			else
+			{
+				FdwXact fx = fdwxacts_failed_to_resolve;
+
+				/* Append the entry at the tail */
+				while (fx->fxact_next != NULL)
+					fx = fx->fxact_next;
+				fx->fxact_next = cur_fdwxact;
+			}
+			LWLockRelease(FdwXactLock);
+
+			fserver = GetForeignServer(cur_fdwxact->serverid);
+			ereport(LOG,
+					(errmsg("could not resolve a foreign transaction on server \"%s\"",
+							fserver->servername),
+					 errdetail("local transaction id is %u, connected by user id %u",
+							   cur_fdwxact->local_xid, cur_fdwxact->userid)));
+		}
+		else
+		{
+			/* Resolved. Update the cache entry if it's valid */
+			if (fdwxact_entry)
+				fdwxact_entry->participants = fdwxact_next;
+
+			elog(DEBUG2, "resolved a foreign transaction xid %u, serverid %d, userid %d",
+				 cur_fdwxact->local_xid, cur_fdwxact->serverid, cur_fdwxact->userid);
+		}
+
+		/* Advance the resolution status to the next */
+		frstate->fdwxact = fdwxact_next;
+	}
+
+	all_resolved = (fdwxacts_failed_to_resolve == NULL);
+
+	if (all_resolved)
+	{
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+		/* Remove the state cache entry from shmem hash table */
+		hash_search(FdwXactStateCache, (void *) &(frstate->waiter->fdwXactWaitXid),
+					HASH_REMOVE, NULL);
+
+		/*
+		 * Remove waiter from shmem queue, if not detached yet. The waiter
+		 * could already be detached if user cancelled to wait before
+		 * resolution.
+		 */
+		if (!SHMQueueIsDetached(&(frstate->waiter->fdwXactLinks)))
+		{
+			TransactionId	wait_xid = frstate->waiter->fdwXactWaitXid;
+
+			SHMQueueDelete(&(frstate->waiter->fdwXactLinks));
+
+			pg_write_barrier();
+
+			/* Set state to complete */
+			frstate->waiter->fdwXactState = FDW_XACT_WAIT_COMPLETE;
+
+			/* Wake up the waiter only when we have set state and removed from queue */
+			SetLatch(&(frstate->waiter->procLatch));
+
+			elog(DEBUG2, "released a proc xid %u", wait_xid);
+		}
+
+		LWLockRelease(FdwXactLock);
+
+		/* Reset resolution state */
+		frstate->waiter = NULL;
+		Assert(frstate->fdwxact == NULL);
+	}
+	else
+	{
+		/*
+		 * Update the fdwxact entry we're processing so that the failed
+		 * fdwxact entries will be processed again.
+		 */
+		frstate->fdwxact = fdwxacts_failed_to_resolve;
+	}
+
+	return all_resolved;
+}
+
+/*
+ * Resolve all dangling foreign transactions on the given database. Get
+ * all dangling foreign transactions from shmem global array and resolve
+ * them one by one.
+ *
+ * Unlike FdwXactResolveDistributedTransaction, for dangling transaction
+ * resolution, we don't bother the order of resolution because these entries
+ * already got out of order. So if failed to resolve a foreign transaction,
+ * we can go to the next foreign transaction that might associates with
+ * an another distributed transaction.
+ */
+void
+FdwXactResolveAllDanglingTransactions(Oid dbid)
+{
+	List		*dangling_fdwxacts = NIL;
+	ListCell	*cell;
+	bool		n_resolved = 0;
+	int			i;
+
+	Assert(OidIsValid(dbid));
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/*
+	 * Walk over the global array to make the list of dangling transactions
+	 * of which corresponding local transaction is on the given database.
+	 */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fxact = FdwXactCtl->fdw_xacts[i];
+
+		/*
+		 * Append the fdwxact entry on the given database to the list if
+		 * it's handled by nobody and the corresponding local transaction
+		 * is not part of the prepared transaction.
+		 */
+		if (fxact->dbid == dbid &&
+			fxact->registered_backend == InvalidBackendId &&
+			!TwoPhaseExists(fxact->local_xid))
+			dangling_fdwxacts = lappend(dangling_fdwxacts, fxact);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/* Return if there is no foreign transaction we need to resolve */
+	if (dangling_fdwxacts == NIL)
+		return;
+
+	foreach(cell, dangling_fdwxacts)
+	{
+		FdwXact fdwxact = (FdwXact) lfirst(cell);
+
+		if (!FdwXactResolveForeignTransaction(fdwxact))
+		{
+			ForeignServer *fserver = GetForeignServer(fdwxact->serverid);
+
+			/*
+			 * If failed to resolve this foreign transaction we skip it in
+			 * this resolution cycle. Try to resolve again in next cycle.
+			 */
+			ereport(LOG,
+					(errmsg("could not resolve a dangling foreign transaction on server \"%s\"",
+							fserver->servername),
+					 errdetail("local transaction id is %u, connected by user id %u",
+							   fdwxact->local_xid, fdwxact->userid)));
+			continue;
+		}
+
+		n_resolved++;
+	}
+
+	list_free(dangling_fdwxacts);
+
+	elog(DEBUG2, "resolved %d dangling foreign xacts", n_resolved);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ *
+ * In commit case, we have already prepared transactions on the foreign
+ * servers during pre-commit. And that prepared transactions will be
+ * resolved by the resolver process. So we don't do anything about the
+ * foreign transaction.
+ *
+ * In abort case, user requested rollback or we changed over rollback
+ * due to error during commit. To close current foreign transaction anyway
+ * we call rollback API to every foreign transaction. If we raised an error
+ * during preparing and came to here, it's possible that some entries of
+ * FdwXactParticipants already registered its FdwXact entry. If there is
+ * we leave them as dangling transaction and ask the resolver process to
+ * process them.
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		int left_fdwxacts = 0;
+
+		foreach (lcell, FdwXactParticipantsForAC)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * Count FdwXact entries that we registered to shared memory array
+			 * in this transaction.
+			 */
+			if (fdw_part->fdw_xact)
+			{
+				/*
+				 * The status of foreign transaction must be either preparing
+				 * or prepared. In any case, since we have registered FdwXact
+				 * entry we leave them to the resolver process. For the preparing
+				 * state, since the foreign transaction might not close yet we
+				 * fall through and call rollback API. For the prepared state,
+				 * since the foreign transaction has closed we don't need to do
+				 * anything.
+				 */
+				Assert(fdw_part->fdw_xact->status == FDW_XACT_PREPARING ||
+					   fdw_part->fdw_xact->status == FDW_XACT_PREPARED);
+
+				left_fdwxacts++;
+				if (fdw_part->fdw_xact->status == FDW_XACT_PREPARED)
+					continue;
+			}
+
+			/*
+			 * Rollback all current foreign transaction. Since we're rollbacking
+			 * the transaction it's too late even if we raise an error here.
+			 * So we log it as warning.
+			 */
+			if (!fdw_part->rollback_foreign_xact(&fdw_part->foreign_xact))
+				ereport(WARNING,
+						(errmsg("could not abort transaction on server \"%s\"",
+								fdw_part->foreign_xact.server->servername)));
+		}
+
+		/* If we left some FdwXact entries, ask the resolver process */
+		if (left_fdwxacts > 0)
+		{
+			ereport(WARNING,
+					(errmsg("left %u foreign transactions in in-doubt status",
+							left_fdwxacts)));
+			fdwxact_maybe_launch_resolver(true);
+		}
+	}
+
+	ForgetAllFdwXactParticipants();
+}
+
+/*
+ * AtPrepare_FdwXacts
+ *
+ * If there are foreign servers involved in the transaction, this function
+ * prepares transactions on those servers.
+ *
+ * Note that it can happen that the transaction aborts after we prepared part
+ * of participants. In this case since we can change to abort we cannot forget
+ * FdwXactParticipantsForAC here. These are processed by the resolver process
+ * during aborting, or at EOXact_FdwXacts.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipantsForAC == NIL)
+		return;
+
+	/*
+	 * We cannot prepare distributed transaction if any foreign server of
+	 * participants in the transaction isn't capable of two-phase commit.
+	 */
+	if ((MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_T_R_INTEGRITY_CONSTRAINT_VIOLATION),
+				 errmsg("can not prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * FdwXactResolveForeignTransaction
+ *
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * handler routine. The foreign transaction can be a dangling transaction
+ * that is not interested by nobody. If the fate of foreign transaction is
+ * not determined yet, it'sdetermined according to the status of corresponding
+ * local transaction.
+ *
+ * If the resolution is successful, remove the foreign transaction entry from
+ * the shared memory and also remove the corresponding on-disk file.
+ */
+static bool
+FdwXactResolveForeignTransaction(FdwXact fdwxact)
+{
+	bool		resolved;
+	bool		is_commit;
+	ForeignServer		*fserver;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	FdwRoutine			*fdw_routine;
+	ForeignTransaction	foreign_xact;
+
+	Assert(fdwxact);
+
+	/*
+	 * Determine whether we commit or abort this foreign transaction.
+	 */
+	if (fdwxact->status == FDW_XACT_COMMITTING_PREPARED)
+		is_commit = true;
+	else if (fdwxact->status == FDW_XACT_ABORTING_PREPARED)
+		is_commit = false;
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	else if (TransactionIdDidCommit(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_COMMITTING_PREPARED;
+		is_commit = true;
+	}
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_ABORTING_PREPARED;
+		is_commit = false;
+	}
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+		is_commit = false;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction
+	 * state is neither committing or aborting. This should not
+	 * happen because we cannot determine to do commit or abort for
+	 * foreign transaction associated with the in-progress local
+	 * transaction.
+	 */
+	else
+		ereport(ERROR,
+				(errmsg("cannot resolve foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid)));
+
+	/* Construct foreign server connection information for passing to API */
+	fserver = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(fserver->fdwid);
+	user_mapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+	foreign_xact.server = fserver;
+	foreign_xact.usermapping = user_mapping;
+	foreign_xact.fx_id = fdwxact->fdw_xact_id;
+
+	/* Resolve the foreign transaction */
+	Assert(fdw_routine->ResolveForeignTransaction);
+	resolved = fdw_routine->ResolveForeignTransaction(&foreign_xact,
+													  is_commit);
+
+	if (!resolved)
+	{
+		ForeignServer *fserver = GetForeignServer(fdwxact->serverid);
+		ereport(ERROR,
+				(errmsg("could not %s a prepared foreign transaction on server \"%s\"",
+						is_commit ? "commit" : "rollback", fserver->servername),
+				 errdetail("local transaction id is %u, connected by user id %u",
+						   fdwxact->local_xid, fdwxact->userid)));
+	}
+	else
+	{
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		if (fdwxact->ondisk)
+			RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+							  fdwxact->serverid, fdwxact->userid,
+							  true);
+		remove_fdw_xact(fdwxact);
+		LWLockRelease(FdwXactLock);
+	}
+
+	return resolved;
+}
+
+/*
+ * Return one FdwXact entry that matches to given arguments, otherwise
+ * return NULL. Since this function search FdwXact entry by unique key
+ * all arguments should be valid.
+ */
+static FdwXact
+get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				 bool need_lock)
+{
+	List	*fdw_xact_list;
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, need_lock);
+
+	/* Could not find entry */
+	if (fdw_xact_list == NIL)
+		return NULL;
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdw_xact_list) == 1);
+
+	return (FdwXact) linitial(fdw_xact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdw_xact_exists(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdw_xact_list;
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, true);
+
+	return fdw_xact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_prepared_fdw_xacts.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdw_xacts(int *length)
+{
+	List		*all_fdw_xacts;
+	ListCell	*lc;
+	FdwXact		fdw_xacts;
+	int			num_fdw_xacts = 0;
+
+	Assert(length != NULL);
+
+	/* Get all entries */
+	all_fdw_xacts = get_fdw_xacts(InvalidOid, InvalidTransactionId,
+								  InvalidOid, InvalidOid, true);
+
+	if (all_fdw_xacts == NIL)
+	{
+		*length = 0;
+		return NULL;
+	}
+
+	fdw_xacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdw_xacts));
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdw_xacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdw_xacts + num_fdw_xacts, fx,
+			   sizeof(FdwXactData));
+		num_fdw_xacts++;
+	}
+
+	*length = num_fdw_xacts;
+	list_free(all_fdw_xacts);
+
+	return fdw_xacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return
+ * NIL.
+ */
+static List*
+get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			  bool need_lock)
+{
+	int i;
+	List	*fdw_xact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact	fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool	matches = true;
+
+		/* xid */
+		if (xid != InvalidTransactionId && xid != fdw_xact->local_xid)
+			matches = false;
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdw_xact->dbid != dbid)
+			matches = false;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdw_xact->serverid)
+			matches = false;
+
+		/* userid */
+		if (OidIsValid(userid) && fdw_xact->userid != userid)
+			matches = false;
+
+		/* Append it if matched */
+		if (matches)
+			fdw_xact_list = lappend(fdw_xact_list, fdw_xact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdw_xact_list;
+}
+
+/*
+ * fdw_xact_redo
+ * Apply the redo log for a foreign transaction.
+ */
+void
+fdw_xact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDW_XACT_REMOVE)
+	{
+		xl_fdw_xact_remove *record = (xl_fdw_xact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. Returned string
+ * value is used to identify foreign transaction. The identifier should not
+ * be same as any other concurrent prepared transaction identifier.
+ *
+ * To make the foreign transactionid, we should ideally use something like
+ * UUID, which gives unique ids with high probability, but that may be expensive
+ * here and UUID extension which provides the function to generate UUID is
+ * not part of the core code.
+ */
+static char *
+generate_fdw_xact_identifier(Oid serverid, Oid userid)
+{
+	char*	fdw_xact_id;
+
+	fdw_xact_id = (char *)palloc(FDW_XACT_ID_MAX_LEN * sizeof(char));
+
+	snprintf(fdw_xact_id, FDW_XACT_ID_MAX_LEN, "%s_%ld_%d_%d",
+			 "fx", Abs(random()), serverid, userid);
+	fdw_xact_id[strlen(fdw_xact_id)] = '\0';
+
+	return fdw_xact_id;
+}
+
+/*
+ * CheckPointFdwXact
+ *
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * In order to avoid disk I/O while holding a light weight lock, the function
+ * first collects the files which need to be synced under FdwXactLock and then
+ * syncs them after releasing the lock. This approach creates a race condition:
+ * after releasing the lock, and before syncing a file, the corresponding
+ * foreign transaction entry and hence the file might get removed. The function
+ * checks whether that's true and ignores the error if so.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdw_xacts = 0;
+
+	/* Quick get-away, before taking lock */
+	if (max_prepared_foreign_xacts <= 0)
+		return;
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/* Another quick, before we allocate memory */
+	if (FdwXactCtl->numFdwXacts <= 0)
+	{
+		LWLockRelease(FdwXactLock);
+		return;
+	}
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		FdwXact		fxact = FdwXactCtl->fdw_xacts[cnt];
+
+		if ((fxact->valid || fxact->inredo) &&
+			!fxact->ondisk &&
+			fxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fxact->dbid, fxact->local_xid,
+								fxact->serverid, fxact->userid,
+								buf, len);
+			fxact->ondisk = true;
+			fxact->insert_start_lsn = InvalidXLogRecPtr;
+			fxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdw_xacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDW_XACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdw_xacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdw_xacts,
+							 serialized_fdw_xacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDW_XACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDW_XACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * ProcessFdwXactBuffer
+ *
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid = ShmemVariableCache->nextXid;
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(insert_start_lsn != InvalidXLogRecPtr);
+
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid, true);
+		if (buf == NULL)
+		{
+			ereport(WARNING,
+					(errmsg("removing corrupt fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+			return NULL;
+		}
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return thecontents in
+ * a structure allocated in-memory. Otherwise return NULL. The structure can
+ * be later freed by the caller.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				bool give_warnings)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+	{
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not stat FDW transaction state file \"%s\": %m",
+							path)));
+		return NULL;
+	}
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdw_xact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+	{
+		CloseTransientFile(fd);
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+		return NULL;
+	}
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+	{
+		CloseTransientFile(fd);
+		return NULL;
+	}
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_READ);
+	if (read(fd, buf, stat.st_size) != stat.st_size)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not read FDW transaction state file \"%s\": %m",
+					  path)));
+		return NULL;
+	}
+
+	pgstat_report_wait_end();
+	CloseTransientFile(fd);
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+	{
+		pfree(buf);
+		return NULL;
+	}
+
+	/* Check if the contents is an expected data */
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fxact_file_data->dbid  != dbid ||
+		fxact_file_data->serverid != serverid ||
+		fxact_file_data->userid != userid ||
+		fxact_file_data->local_xid != xid)
+	{
+		ereport(WARNING,
+			(errmsg("invalid foreign transaction state file \"%s\"",
+					path)));
+		CloseTransientFile(fd);
+		pfree(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+/*
+ * PrescanFdwXacts
+ *
+ * Scan the all foreign transactions directory for oldest active transaction.
+ * This is run during database startup, after we completed reading WAL.
+ * ShmemVariableCache->nextXid has been set to one more than the highest XID
+ * for which evidence exists in WAL.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	TransactionId nextXid = ShmemVariableCache->nextXid;
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+		 strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			TransactionId local_xid;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/*
+			 * Remove a foreign prepared transaction file corresponding to an
+			 * XID, which is too new.
+			 */
+			if (TransactionIdFollowsOrEquals(local_xid, nextXid))
+			{
+				ereport(WARNING,
+						(errmsg("removing future foreign prepared transaction file \"%s\"",
+								clde->d_name)));
+				RemoveFdwXactFile(dbid, local_xid, serverid, userid, true);
+				continue;
+			}
+
+			if (TransactionIdPrecedesOrEquals(local_xid, oldestActiveXid))
+				oldestActiveXid = local_xid;
+		}
+	}
+
+	FreeDir(cldir);
+	return oldestActiveXid;
+}
+
+/*
+ * restoreFdwXactData
+ *
+ * Scan pg_fdw_xact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid, bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * FdwXactRedoAdd
+ *
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fxact = insert_fdw_xact(fxact_data->dbid, fxact_data->local_xid,
+							fxact_data->serverid, fxact_data->userid,
+							fxact_data->fdw_xact_id);
+
+	/*
+	 * Set status as preparing, since we do not know the xact status
+	 * right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->insert_start_lsn = start_lsn;
+	fxact->insert_end_lsn = end_lsn;
+	fxact->inredo = true;	/* added in redo */
+	fxact->valid = false;
+	fxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * FdwXactRedoRemove
+ *
+ * Remove the corresponding fdw_xact entry from FdwXactCtl.
+ * Also remove fdw_xact file if a foreign transaction was saved
+ * via an earlier checkpoint.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdw_xact(dbid, xid, serverid, userid,
+							   false);
+
+	if (fdwxact == NULL)
+		return;
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdw_xact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(bool *newval, void **extra, GucSource source)
+{
+	/* Parameter check */
+	if (*newval &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_xacts\" or \"max_foreign_xact_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdw_xacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_prepared_fdw_xacts(PG_FUNCTION_ARGS)
+{
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdw_xacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdw_xacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(6, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdw_xacts = get_all_fdw_xacts(&num_fdw_xacts);
+		status->num_xacts = num_fdw_xacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdw_xact = &status->fdw_xacts[status->cur_xact++];
+		Datum		values[6];
+		bool		nulls[6];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdw_xact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdw_xact->dbid);
+		values[1] = TransactionIdGetDatum(fdw_xact->local_xid);
+		values[2] = ObjectIdGetDatum(fdw_xact->serverid);
+		values[3] = ObjectIdGetDatum(fdw_xact->userid);
+		switch (fdw_xact->status)
+		{
+			case FDW_XACT_PREPARING:
+				xact_status = "prepared";
+				break;
+			case FDW_XACT_COMMITTING_PREPARED:
+				xact_status = "committing";
+				break;
+			case FDW_XACT_ABORTING_PREPARED:
+				xact_status = "aborting";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		/* should this be really interpreted by FDW */
+		values[5] = PointerGetDatum(cstring_to_text_with_len(fdw_xact->fdw_xact_id,
+															 strlen(fdw_xact->fdw_xact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+	bool			ret;
+
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, true);
+	if (fdwxact == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("could not find foreign transaction entry"))));
+
+	ret = FdwXactResolveForeignTransaction(fdwxact);
+
+	PG_RETURN_BOOL(ret);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, false);
+	if (fdwxact == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("could not find foreign transaction entry"))));
+
+	remove_fdw_xact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_VOID();
+}
diff --git a/src/backend/foreign/fdwxact_launcher.c b/src/backend/foreign/fdwxact_launcher.c
new file mode 100644
index 0000000..6782c33
--- /dev/null
+++ b/src/backend/foreign/fdwxact_launcher.c
@@ -0,0 +1,587 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * There is a shared memory area where the information of resolver process
+ * is stored. Requesting of starting new resolver process by backend process
+ * is done via that shared memory area. Note that the launcher is assuming
+ * that there is no more than one starting request for a database.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/foreign/fdwxact_launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "foreign/fdwxact.h"
+#include "foreign/fdwxact_launcher.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/resolver_internal.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid, int slot);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+Datum pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS);
+
+/*
+ * Wake up the launcher process.
+ */
+void
+FdwXactLauncherWakeup(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR1);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->FdwXactQueue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+
+		now = GetCurrentTimestamp();
+
+		if (TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			bool launched;
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+				last_start_time = now;
+		}
+		else
+		{
+			/*
+			 * The wint in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver worker
+ * if not running yet. A foreign transaction resolver worker is responsible
+ * for resolution of foreign transaction that are registered on a database.
+ * So if a resolver worker already is launched, we don't need to launch new
+ * one.
+ */
+void
+fdwxact_maybe_launch_resolver(bool ignore_error)
+{
+	FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->pid != InvalidPid &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/*
+	 * If we found the resolver for my database, we don't need to launch new
+	 * one but wake running worker up.
+	 */
+	if (found)
+	{
+		SetLatch(resolver->latch);
+
+		elog(DEBUG1, "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		return;
+	}
+
+	/* Looking for unused worker slot */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	/*
+	 * However if there are no more free worker slots, inform user about it before
+	 * exiting.
+	 */
+	if (!found)
+	{
+		LWLockRelease(FdwXactResolverLock);
+
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+		return;
+	}
+
+	Assert(resolver->pid == InvalidPid);
+
+	/* Found a new resolver process */
+	resolver->dbid = MyDatabaseId;
+	resolver->in_use = true;
+
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Wake up launcher */
+	FdwXactLauncherWakeup();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid' at 'slot' if given. If slot is negative value we find an unused slot.
+ * Note that caller must hold FdwXactResolverLock in exclusive mode.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid, int slot)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int launch_slot = slot;
+
+	/* If slot number is invalid, we find an unused slot */
+	if (launch_slot < 0)
+	{
+		int i;
+
+		for (i = 0; i < max_foreign_xact_resolvers; i++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+			if (resolver->in_use && resolver->dbid == dbid)
+				return;
+
+			if (!resolver->in_use)
+			{
+				launch_slot = i;
+				break;
+			}
+		}
+	}
+
+	/* No unused found */
+	if (launch_slot < 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[launch_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_main_arg = Int32GetDatum(launch_slot);
+	bgw.bgw_notify_pid = (Datum) 0;
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch all foreign transaction resolvers that are required by backend process
+ * but not running.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	int i, j;
+	int num_launches = 0;
+	int num_unused_slots = 0;
+	int num_dbs = 0;
+	bool launched = false;
+	Oid	*dbs_to_launch;
+	Oid *dbs_having_worker = palloc0(sizeof(Oid) * max_foreign_xact_resolvers);
+
+	/*
+	 * Launch resolver workers on the databases that are requested
+	 * by backend processes.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* Remember unused worker slots */
+		if (!resolver->in_use)
+			num_unused_slots++;
+
+		/* Remember databases that are having a resolve worker */
+		if (OidIsValid(resolver->dbid))
+			dbs_having_worker[num_dbs++] = resolver->dbid;
+
+		/* Launch new foreign transaction resolver worker on the database */
+		if (resolver->in_use &&
+			OidIsValid(resolver->dbid) &&
+			resolver->pid == InvalidPid)
+		{
+			fdwxact_launch_resolver(resolver->dbid, i);
+			launched = true;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* There is no unused slot, exit */
+	if (num_unused_slots == 0)
+		return launched;
+
+	dbs_to_launch = (Oid *) palloc(sizeof(Oid) * num_unused_slots);
+
+	/*
+	 * If there is unused slot, we can launch foreign transaction resolver
+	 * on databases that has unresolved foreign transaction but doesn't
+	 * have any resolver. This usually happens when resolvers crash for
+	 * whatever reason. Scanning all FdwXact entries could takes time but
+	 * since this is a relaunch case it's not harmless.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool found = false;
+
+		if (num_launches > num_unused_slots)
+			break;
+
+		for (j = 0; j < num_dbs; j++)
+		{
+			if (dbs_having_worker[j] == fdw_xact->dbid)
+			{
+				found = true;
+				break;
+			}
+		}
+
+		if (found)
+			continue;
+
+		dbs_to_launch[num_launches++] = fdw_xact->dbid;
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* Launch resolver process for a database at any worker slot */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < num_launches; i++)
+	{
+		fdwxact_launch_resolver(dbs_to_launch[i], -1);
+		launched = true;
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+
+/*
+ * Returns activity of foreign transaction resolvers, including pids, the number
+ * of tasks and the last resolution time.
+ */
+Datum
+pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		if (resolver->pid == 0)
+		{
+			SpinLockRelease(&(MyFdwXactResolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/foreign/fdwxact_resolver.c b/src/backend/foreign/fdwxact_resolver.c
new file mode 100644
index 0000000..7f7ff8f
--- /dev/null
+++ b/src/backend/foreign/fdwxact_resolver.c
@@ -0,0 +1,310 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for every databases.
+ *
+ * A resolver process continues to resolve foreign transactions on a database
+ * It resolves two types of foreign transactions: on-line foreign transaction
+ * and dangling foreign transaction. The on-line foreign transaction is a
+ * foreign transaction that a concurrent backend process is waiting for
+ * resolution. The dangling transaction is a foreign transaction that corresponding
+ * distributed transaction ended up in in-doubt state. A resolver process
+ * doesn' exit as long as there is at least one unresolved foreign transaction
+ * on the database even if the timeout has come.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/foreign/fdwxact_resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "foreign/fdwxact.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/fdwxact_launcher.h"
+#include "foreign/resolver_internal.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+
+//static MemoryContext ResolveContext = NULL;
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FdwXactRslvLoop(void);
+static long FdwXactRslvComputeSleepTime(TimestampTz now);
+static void FdwXactRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+	FdwXactLauncherWakeup();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	TIMESTAMP_NOBEGIN(MyFdwXactResolver->last_resolved_time);
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FdwXactRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FdwXactRslvLoop(void)
+{
+	FdwXactResolveState *fstate;
+
+	/* Create an FdwXactResolveState */
+	fstate = CreateFdwXactResolveState();
+
+	/* Enter main loop */
+	for (;;)
+	{
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time;
+		bool		resolved;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		/* Resolve a distributed transaction */
+		StartTransactionCommand();
+		resolved = FdwXactResolveDistributedTransaction(fstate);
+		CommitTransactionCommand();
+
+		now = GetCurrentTimestamp();
+
+		/* Update my state */
+		if (resolved)
+			MyFdwXactResolver->last_resolved_time = now;
+
+		/* Check for fdwxact resolver timeout */
+		FdwXactRslvCheckTimeout(now);
+
+		/*
+		 * If we have resolved any distributed transaction we go the next
+		 * without both resolving dangling transaction and sleeping because
+		 * there might be other on-line transactions waiting to be resolved.
+		 */
+		if (!resolved)
+		{
+			/* Resolve dangling transactions as mush as possible */
+			StartTransactionCommand();
+			FdwXactResolveAllDanglingTransactions(MyDatabaseId);
+			CommitTransactionCommand();
+
+			sleep_time = FdwXactRslvComputeSleepTime(now);
+
+			rc = WaitLatch(MyLatch,
+						   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+						   sleep_time,
+						   WAIT_EVENT_FDW_XACT_RESOLVER_MAIN);
+
+			if (rc & WL_POSTMASTER_DEATH)
+				proc_exit(1);
+		}
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FdwXactRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	/*
+	 * Reached to the timeout. We exit if there is no more both pending on-line
+	 * transactions and dangling transactions.
+	 */
+	if (!fdw_xact_exists(InvalidTransactionId, MyDatabaseId, InvalidOid,
+						 InvalidOid))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyFdwXactResolver->dbid))));
+		CommitTransactionCommand();
+
+		fdwxact_resolver_detach();
+		proc_exit(0);
+	}
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. Return the sleep time
+ * in milliseconds, -1 means that we reached to the timeout and should exits
+ */
+static long
+FdwXactRslvComputeSleepTime(TimestampTz now)
+{
+	static TimestampTz	wakeuptime = 0;
+	long	sleeptime;
+	long	sec_to_timeout;
+	int		microsec_to_timeout;
+
+	if (now >= wakeuptime)
+		wakeuptime = TimestampTzPlusMilliseconds(now,
+												 foreign_xact_resolution_retry_interval);
+
+	/* Compute relative time until wakeup. */
+	TimestampDifference(now, wakeuptime,
+						&sec_to_timeout, &microsec_to_timeout);
+
+	sleeptime = sec_to_timeout * 1000 + microsec_to_timeout / 1000;
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index eac78a5..1873a24 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -155,6 +155,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMapping - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index d2b695e..9243686 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -16,6 +16,8 @@
 
 #include "libpq/pqsignal.h"
 #include "access/parallel.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/fdwxact_launcher.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "port/atomics.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 8a5b2b3..a67b34d 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3492,6 +3492,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_LAUNCHER_MAIN:
 			event_name = "LogicalLauncherMain";
 			break;
@@ -3683,6 +3689,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -3898,6 +3907,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDW_XACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 41de140..138dae4 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -100,6 +100,8 @@
 #include "common/file_perm.h"
 #include "common/ip.h"
 #include "common/string.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/fdwxact_launcher.h"
 #include "lib/ilist.h"
 #include "libpq/auth.h"
 #include "libpq/libpq.h"
@@ -905,6 +907,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires maX_foreign_xact_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -980,12 +986,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index afb4972..960fd6a 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -154,6 +154,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDW_XACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 0c86a58..5f321fe 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -22,6 +22,7 @@
 #include "access/subtrans.h"
 #include "access/twophase.h"
 #include "commands/async.h"
+#include "foreign/fdwxact_launcher.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "postmaster/autovacuum.h"
@@ -150,6 +151,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
 		size = add_size(size, BackendRandomShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -270,6 +273,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 	SyncScanShmemInit();
 	AsyncShmemInit();
 	BackendRandomShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index bf2f4db..461ba5c 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -90,6 +90,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -245,6 +247,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1312,6 +1315,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	volatile TransactionId replication_slot_xmin = InvalidTransactionId;
 	volatile TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	volatile TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1373,6 +1377,7 @@ GetOldestXmin(Relation rel, int flags)
 	/* fetch into volatile var while ProcArrayLock is held */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1423,6 +1428,15 @@ GetOldestXmin(Relation rel, int flags)
 		result = replication_slot_xmin;
 
 	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDW_XACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
+	/*
 	 * After locks have been released and defer_cleanup_age has been applied,
 	 * check whether we need to back up further to make logical decoding
 	 * possible. We need to do so if we're computing the global limit (rel =
@@ -2999,6 +3013,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index e6025ec..a42d06e 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -50,3 +50,5 @@ OldSnapshotTimeMapLock				42
 BackendRandomLock					43
 LogicalRepWorkerLock				44
 CLogTruncationLock					45
+FdwXactLock					46
+FdwXactResolverLock			47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 6f9aaa5..ec09515 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -38,6 +38,7 @@
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
+#include "foreign/fdwxact.h"
 #include "miscadmin.h"
 #include "pgstat.h"
 #include "postmaster/autovacuum.h"
@@ -398,6 +399,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* initialize fields for fdw xact */
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -799,6 +804,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index e4c6e3d..f09955f 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -43,6 +43,8 @@
 #include "commands/async.h"
 #include "commands/prepare.h"
 #include "executor/spi.h"
+#include "foreign/fdwxact_resolver.h"
+#include "foreign/fdwxact_launcher.h"
 #include "jit/jit.h"
 #include "libpq/libpq.h"
 #include "libpq/pqformat.h"
@@ -2971,6 +2973,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0bec391..121d7bf 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -42,6 +42,7 @@
 #include "commands/variable.h"
 #include "commands/trigger.h"
 #include "common/string.h"
+#include "foreign/fdwxact.h"
 #include "funcapi.h"
 #include "jit/jit.h"
 #include "libpq/auth.h"
@@ -660,6 +661,10 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -1832,6 +1837,16 @@ static struct config_bool ConfigureNamesBool[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+			gettext_noop("Sets the usage of two-phase commit protocol for distributed transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		false,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
 	/* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL
@@ -2236,6 +2251,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 4e61bc6..88cdc85 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -121,6 +121,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -287,6 +289,20 @@
 
 
 #------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
+#------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index ad06e8e..ca3eb62 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index ab5cb7f..609578c 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -209,6 +209,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdw_xact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 895a51f..5f0683d 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -306,6 +306,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_worker_processes);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_xacts setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 6fb403a..6d867c8 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -730,6 +730,7 @@ GuessControlValues(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -957,6 +958,7 @@ RewriteControlFile(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* Contents are protected with a CRC */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca..15bfeb4 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -26,6 +26,7 @@
 #include "commands/dbcommands_xlog.h"
 #include "commands/sequence.h"
 #include "commands/tablespace.h"
+#include "foreign/fdwxact_xlog.h"
 #include "replication/message.h"
 #include "replication/origin.h"
 #include "rmgrdesc.h"
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 0bbe9879..c15dff7 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDW_XACT_ID, "Foreign Transactions", fdw_xact_redo, fdw_xact_desc, fdw_xact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 0e932da..b199c88 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 				TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index c7b4144..7180bd1 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -105,6 +105,13 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE				(1U << 3)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 30610b3..795e85a 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -227,6 +227,7 @@ typedef struct xl_parameter_change
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index 773d9e6..3d5333a 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -178,6 +178,7 @@ typedef struct ControlFileData
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 8e4145f..21e5bcc 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5199,6 +5199,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '6053', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_fdwxact_resolver', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,oid,timestamptz}',
+  proargmodes => '{o,o,o,o}',
+  proargnames => '{pid,dbid,n_entries,last_resolved_time}',
+  prosrc => 'pg_stat_get_fdwxact_resolver' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5910,6 +5917,22 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '6050', descr => 'view foreign transactions',
+  proname => 'pg_prepared_fdw_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,text}',
+  proargmodes => '{o,o,o,o,o,o}',
+  proargnames => '{dbid,transaction,serverid,userid,status,identifier}',
+  prosrc => 'pg_prepared_fdw_xacts' },
+{ oid => '6051', descr => 'remove foreign transaction',
+  proname => 'pg_remove_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_remove_fdw_xact' },
+{ oid => '6052', descr => 'resolve foreign transaction',
+  proname => 'pg_resolve_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_resolve_fdw_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index c14eb54..f76e83d 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -13,6 +13,7 @@
 #define FDWAPI_H
 
 #include "access/parallel.h"
+#include "foreign/fdwxact.h"
 #include "nodes/execnodes.h"
 #include "nodes/relation.h"
 
@@ -168,6 +169,12 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef bool (*PrepareForeignTransaction_function) (ForeignTransaction *foreign_xact);
+typedef bool (*CommitForeignTransaction_function) (ForeignTransaction *foreign_xact);
+typedef bool (*RollbackForeignTransaction_function) (ForeignTransaction *foreing_xact);
+typedef bool (*ResolveForeignTransaction_function) (ForeignTransaction *foreign_xact,
+													bool is_commit);
+typedef bool (*IsTwoPhaseCommitEnabled_function) (Oid serverid);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -235,6 +242,13 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for distributed transactions */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	ResolveForeignTransaction_function ResolveForeignTransaction;
+	IsTwoPhaseCommitEnabled_function IsTwoPhaseCommitEnabled;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
@@ -247,7 +261,6 @@ typedef struct FdwRoutine
 	ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild;
 } FdwRoutine;
 
-
 /* Functions in foreign/foreign.c */
 extern FdwRoutine *GetFdwRoutine(Oid fdwhandler);
 extern Oid	GetForeignServerIdByRelId(Oid relid);
@@ -258,4 +271,7 @@ extern bool IsImportableForeignTable(const char *tablename,
 						 ImportForeignSchemaStmt *stmt);
 extern Path *GetExistingLocalJoinPath(RelOptInfo *joinrel);
 
+/* Functions in foreign/fdwxact.c */
+extern void FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, char *fx_id);
+
 #endif							/* FDWAPI_H */
diff --git a/src/include/foreign/fdwxact.h b/src/include/foreign/fdwxact.h
new file mode 100644
index 0000000..5138a2c
--- /dev/null
+++ b/src/include/foreign/fdwxact.h
@@ -0,0 +1,147 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL distributed transaction manager
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/foreign/fdwxact.h
+ */
+#ifndef FDW_XACT_H
+#define FDW_XACT_H
+
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "foreign/fdwxact_xlog.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+#define	FDW_XACT_NOT_WAITING		0
+#define	FDW_XACT_WAITING			1
+#define	FDW_XACT_WAIT_COMPLETE		2
+
+#define FdwXactEnabled() (max_prepared_foreign_xacts > 0)
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDW_XACT_ID_MAX_LEN 200
+
+/* Enum to track the status of prepared foreign transaction */
+typedef enum
+{
+	FDW_XACT_INITIAL,
+	FDW_XACT_PREPARING,					/* foreign transaction is being prepared */
+	FDW_XACT_PREPARED,					/* foriegn transaction is prepared */
+	FDW_XACT_COMMITTING_PREPARED,		/* foreign prepared transaction is to
+										 * be committed */
+	FDW_XACT_ABORTING_PREPARED, /* foreign prepared transaction is to be
+								 * aborted */
+} FdwXactStatus;
+
+/* Shared memory entry for a prepared or being prepared foreign transaction */
+typedef struct FdwXactData *FdwXact;
+
+typedef struct FdwXactData
+{
+	FdwXact		fxact_free_next;	/* Next free FdwXact entry */
+	FdwXact		fxact_next;		/* Pointer to the neext FdwXact entry accosiated
+								 * with the same transaction */
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	TransactionId local_xid;	/* XID of local transaction */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	FdwXactStatus status;		/* The state of the foreign
+								 * transaction. This doubles as the
+								 * action to be taken on this entry. */
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;		/* XLOG offset of inserting this entry start */
+	XLogRecPtr	insert_end_lsn;		/* XLOG offset of inserting this entry end */
+
+	bool		valid; /* Has the entry been complete and written to file? */
+	BackendId	registered_backend;	/* Backend who registered this entry */
+	bool		ondisk;			/* TRUE if prepare state file is on disk */
+	bool		inredo;			/* TRUE if entry was added via xlog_redo */
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN];		/* prepared transaction identifier */
+} FdwXactData;
+
+/* Shared memory layout for maintaining foreign prepared transaction entries. */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		freeFdwXacts;
+
+	/* Number of valid foreign transaction entries */
+	int			numFdwXacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdw_xacts[FLEXIBLE_ARRAY_MEMBER];		/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* Struct for foreign transaction resolution */
+typedef struct FdwXactResolveState
+{
+	Oid				dbid;		/* database oid */
+	TransactionId	wait_xid;	/* local transaction id waiting to be resolved */
+	PGPROC			*waiter;	/* backend process waiter */
+	FdwXact			fdwxact;	/* foreign transaction entries to resolve */
+} FdwXactResolveState;
+
+/* Struct for foreign transaction passed to API */
+typedef struct ForeignTransaction
+{
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	char			*fx_id;
+} ForeignTransaction;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern bool foreign_twophase_commit;
+
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern bool fdw_xact_exists(TransactionId xid, Oid dboid, Oid serverid,
+				Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern void PreCommit_FdwXacts(void);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern bool FdwXactResolveDistributedTransaction(FdwXactResolveState *fstate);
+extern void FdwXactResolveAllDanglingTransactions(Oid dbid);
+extern bool ForeignTwophaseCommitRequired(void);
+extern FdwXactResolveState *CreateFdwXactResolveState(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void FdwXactMarkForeignTransactionModified(ResultRelInfo *resultRelInfo,
+												  int flags);
+extern bool check_foreign_twophase_commit(bool *newval, void **extra,
+										  GucSource source);
+
+#endif   /* FDW_XACT_H */
diff --git a/src/include/foreign/fdwxact_launcher.h b/src/include/foreign/fdwxact_launcher.h
new file mode 100644
index 0000000..6ed003b
--- /dev/null
+++ b/src/include/foreign/fdwxact_launcher.h
@@ -0,0 +1,31 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/foreign/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _FDWXACT_LAUNCHER_H
+#define _FDWXACT_LAUNCHER_H
+
+#include "foreign/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherWakeup(void);
+
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+
+extern bool IsFdwXactLauncher(void);
+
+extern void fdwxact_maybe_launch_resolver(bool ignore_error);
+
+
+#endif	/* _FDWXACT_LAUNCHER_H */
diff --git a/src/include/foreign/fdwxact_resolver.h b/src/include/foreign/fdwxact_resolver.h
new file mode 100644
index 0000000..5afd98c
--- /dev/null
+++ b/src/include/foreign/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/foreign/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "foreign/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int foreign_xact_resolver_timeout;
+
+#endif		/* FDWXACT_RESOLVER_H */
diff --git a/src/include/foreign/fdwxact_xlog.h b/src/include/foreign/fdwxact_xlog.h
new file mode 100644
index 0000000..f42725e
--- /dev/null
+++ b/src/include/foreign/fdwxact_xlog.h
@@ -0,0 +1,51 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/foreign/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDW_XACT_INSERT	0x00
+#define XLOG_FDW_XACT_REMOVE	0x10
+
+/* Same as GIDSIZE */
+#define FDW_XACT_MAX_ID_LEN 200
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdw_xact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+} xl_fdw_xact_remove;
+
+extern void fdw_xact_redo(XLogReaderState *record);
+extern void fdw_xact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdw_xact_identify(uint8 info);
+
+#endif	/* FDWXACT_XLOG_H */
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 3ca12e6..d030368 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -68,10 +68,10 @@ typedef struct ForeignTable
 	List	   *options;		/* ftoptions as DefElem list */
 } ForeignTable;
 
-
 extern ForeignServer *GetForeignServer(Oid serverid);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperByName(const char *name,
 							bool missing_ok);
diff --git a/src/include/foreign/resolver_internal.h b/src/include/foreign/resolver_internal.h
new file mode 100644
index 0000000..9f8676b
--- /dev/null
+++ b/src/include/foreign/resolver_internal.h
@@ -0,0 +1,65 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/foreign/resovler_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _RESOLVER_INTERNAL_H
+#define _RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t	pid;	/* this resolver's PID, or 0 if not active */
+	Oid		dbid;	/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool	in_use;
+
+	/* Stats */
+	TimestampTz	last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t	mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	*latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/*
+	 * Foreign transaction resolution queue. Protected by FdwXactLock.
+	 */
+	SHM_QUEUE	FdwXactQueue;
+
+	/* Supervisor process */
+	pid_t		launcher_pid;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif	/* _RESOLVER_INTERNAL_H */
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index d59c24a..f74d1be 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -759,6 +759,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDW_XACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -832,7 +834,8 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDW_XACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -912,6 +915,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_READ,
+	WAIT_EVENT_FDW_XACT_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index cb613c8..45880b2 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -153,6 +153,16 @@ struct PGPROC
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction
+								 * resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+
+	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
 	 * their lock.
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index 75bab29..25d6a2f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDW_XACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -124,4 +126,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 								TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 668d9ef..81560bd 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -94,6 +94,8 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 078129f..31502a0 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1413,6 +1413,13 @@ pg_policies| SELECT n.nspname AS schemaname,
    FROM ((pg_policy pol
      JOIN pg_class c ON ((c.oid = pol.polrelid)))
      LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace)));
+pg_prepared_fdw_xacts| SELECT f.dbid,
+    f.transaction,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.identifier
+   FROM pg_prepared_fdw_xacts() f(dbid, transaction, serverid, userid, status, identifier);
 pg_prepared_statements| SELECT p.name,
     p.statement,
     p.prepare_time,
@@ -1821,6 +1828,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_fdwxact_resolvers| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_fdwxact_resolver() r(pid, dbid, n_entries, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_progress_vacuum| SELECT s.pid,
     s.datid,
     d.datname,
-- 
2.10.5

From 555fec86f082a092725fbae1c85a4e00d70f8539 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Thu, 8 Feb 2018 11:26:46 +0900
Subject: [PATCH v19 1/4] Keep track of writing on non-temporary relation.

---
 src/backend/access/heap/heapam.c | 12 ++++++++++++
 src/include/access/xact.h        |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index fb63471..c2db19b 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2629,6 +2629,10 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 		heap_freetuple(heaptup);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleGetOid(tup);
 }
 
@@ -3453,6 +3457,10 @@ l1:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleMayBeUpdated;
 }
 
@@ -4403,6 +4411,10 @@ l2:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	bms_free(hot_attrs);
 	bms_free(proj_idx_attrs);
 	bms_free(key_attrs);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 689c57c..2c1b2d8 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -98,6 +98,11 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
-- 
2.10.5

From edcdf0a8b9f780b9fdb9a0430ee88a007975399a Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:46:01 +0900
Subject: [PATCH v19 3/4] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/connection.c              | 538 +++++++++++++++++++------
 contrib/postgres_fdw/expected/postgres_fdw.out | 387 +++++++++++++++++-
 contrib/postgres_fdw/option.c                  |   5 +-
 contrib/postgres_fdw/postgres_fdw.c            |  60 ++-
 contrib/postgres_fdw/postgres_fdw.h            |  10 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql      | 151 ++++++-
 doc/src/sgml/postgres-fdw.sgml                 |  37 ++
 7 files changed, 1044 insertions(+), 144 deletions(-)

diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index fe4893a..28f87a6 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -14,9 +14,12 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
-#include "catalog/pg_user_mapping.h"
 #include "access/xact.h"
+#include "catalog/pg_user_mapping.h"
+#include "commands/defrem.h"
+#include "foreign/fdwapi.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -56,6 +59,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		am_participant_of_ac;	/* true if fdwxact code control the transaction */
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -78,7 +82,7 @@ static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_xact_callback(XactEvent event, void *arg);
 static void pgfdw_subxact_callback(SubXactEvent event,
 					   SubTransactionId mySubid,
@@ -91,20 +95,14 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 						 bool ignore_errors);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 						 PGresult **result);
-
-
-/*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
- */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static bool pgfdw_commit_transaction(ConnCacheEntry *entry);
+static bool pgfdw_rollback_transaction(ConnCacheEntry *entry);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
 {
 	bool		found;
 	ConnCacheEntry *entry;
@@ -136,11 +134,8 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
 	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
+	key = umid;
 
 	/*
 	 * Find or create cached entry for requested connection.
@@ -182,6 +177,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping		*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +186,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->am_participant_of_ac = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +197,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,16 +213,46 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
 /*
  * Connect to remote server using specified server and user mapping properties.
+ * If the attempt to connect fails, and the caller can handle connection failure
+ * (connection_error_ok = true) return NULL, throw error otherwise.
  */
 static PGconn *
 connect_pg_server(ForeignServer *server, UserMapping *user)
@@ -265,11 +301,22 @@ connect_pg_server(ForeignServer *server, UserMapping *user)
 
 		conn = PQconnectdbParams(keywords, values, false);
 		if (!conn || PQstatus(conn) != CONNECTION_OK)
+		{
+			char	   *connmessage;
+			int			msglen;
+
+			/* libpq typically appends a newline, strip that */
+			connmessage = pstrdup(PQerrorMessage(conn));
+			msglen = strlen(connmessage);
+			if (msglen > 0 && connmessage[msglen - 1] == '\n')
+				connmessage[msglen - 1] = '\0';
+
 			ereport(ERROR,
 					(errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION),
 					 errmsg("could not connect to server \"%s\"",
 							server->servername),
 					 errdetail_internal("%s", pchomp(PQerrorMessage(conn)))));
+		}
 
 		/*
 		 * Check that non-superuser has used password to establish connection;
@@ -414,15 +461,24 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
+	ForeignServer	*server = GetForeignServer(serverid);
 
 	/* Start main transaction if we haven't yet */
 	if (entry->xact_depth <= 0)
 	{
 		const char *sql;
 
+		/* Register the new foreign server if enabled */
+		if (server_uses_twophase_commit(server))
+		{
+			/* Register foreign server with auto-generated identifer */
+			FdwXactRegisterForeignTransaction(serverid, userid, NULL);
+			entry->am_participant_of_ac = true;
+		}
+
 		elog(DEBUG3, "starting remote transaction on connection %p",
 			 entry->conn);
 
@@ -650,12 +706,11 @@ static void
 pgfdw_xact_callback(XactEvent event, void *arg)
 {
 	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
+	ConnCacheEntry	*entry;
 
-	/* Quick exit if no connections were touched in this transaction. */
+	/* Quick exit if no connections were touched in this transaction */
 	if (!xact_got_connection)
 		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote transactions, and
 	 * close them.
@@ -663,17 +718,20 @@ pgfdw_xact_callback(XactEvent event, void *arg)
 	hash_seq_init(&scan, ConnectionHash);
 	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
 	{
-		PGresult   *res;
-
 		/* Ignore cache entry if no open connection right now */
 		if (entry->conn == NULL)
 			continue;
 
+		/*
+		 * Foreign transactions participating to atomic commit are ended
+		 * by two-phase commit APIs. Ignore them.
+		 */
+		if (entry->am_participant_of_ac)
+			continue;
+
 		/* If it has an open remote transaction, try to close it */
 		if (entry->xact_depth > 0)
 		{
-			bool		abort_cleanup_failure = false;
-
 			elog(DEBUG3, "closing remote transaction on connection %p",
 				 entry->conn);
 
@@ -681,40 +739,7 @@ pgfdw_xact_callback(XactEvent event, void *arg)
 			{
 				case XACT_EVENT_PARALLEL_PRE_COMMIT:
 				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
+					pgfdw_commit_transaction(entry);
 					break;
 				case XACT_EVENT_PRE_PREPARE:
 
@@ -739,66 +764,7 @@ pgfdw_xact_callback(XactEvent event, void *arg)
 					break;
 				case XACT_EVENT_PARALLEL_ABORT:
 				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
+					pgfdw_rollback_transaction(entry);
 					break;
 			}
 		}
@@ -1193,3 +1159,329 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * The function prepares transaction on foreign server. This function
+ * is called only at the pre-commit phase of the local transaction. Since
+ * we should have the connection to the server that we are interested in
+ * we don't use serverid and userid that are necessary to get user mapping
+ * that is the key of the connection cache.
+ */
+bool
+postgresPrepareForeignTransaction(FdwXactResolveState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+	PGresult	*res;
+	StringInfo	command;
+
+	entry = hash_search(ConnectionHash, &(state->umid), HASH_FIND, NULL);
+
+	if (!entry->conn)
+		return false;
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	if (result)
+		elog(DEBUG1, "prepared foreign transaction on server %u with ID %s",
+			 state->serverid, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/*
+ * The function commits the transactionon foreign server. This
+ * function is called both at the pre-commit phase of the local transaction
+ * when committing and at the end of the local transaction when aborting.
+ * Since we should the connections to the server that involved with the local
+ * transaction we don't use serverid and userid that are necessary to get
+ * user mapping that is the key of connection cache.
+ */
+bool
+postgresCommitForeignTransaction(FdwXactResolveState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+
+	entry = hash_search(ConnectionHash, &(state->umid),
+						HASH_FIND, NULL);
+
+	result = pgfdw_commit_transaction(entry);
+
+	return result;
+}
+
+/*
+ * The function rollbacks the transactionon foreign server. This
+ * function is called both at the pre-commit phase of the local transaction
+ * when committing and at the end of the local transaction when aborting.
+ * Since we should the connections to the server that involved with the local
+ * transaction we don't use serverid and userid that are necessary to get
+ * user mapping that is the key of connection cache.
+ */
+bool
+postgresRollbackForeignTransaction(FdwXactResolveState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool ret;
+
+	entry = hash_search(ConnectionHash, &(state->umid),
+						HASH_FIND, NULL);
+
+	/* Rollback a remote transaction */
+	ret = pgfdw_rollback_transaction(entry);
+
+	return ret;
+}
+
+bool
+postgresResolveForeignTransaction(FdwXactResolveState *state, bool is_commit)
+{
+	ConnCacheEntry *entry = NULL;
+	StringInfo	command;
+	bool result;
+	PGresult	*res;
+
+	entry = GetConnectionState(state->umid, false, false);
+
+	if (!entry->conn)
+		return false;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 state->fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		/*
+		 * The command failed, raise a warning to log the reason of failure.
+		 * We may not be in a transaction here, so raising error doesn't
+		 * help. Even if we are in a transaction, it would be the resolver
+		 * transaction, which will get aborted on raising error, thus
+		 * delaying resolution of other prepared foreign transactions.
+		 */
+		pgfdw_report_error(LOG, res, entry->conn, false, command->data);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * If we tried to COMMIT/ABORT a prepared transaction and the prepared
+		 * transaction was missing on the foreign server, it was probably
+		 * resolved by some other means. Anyway, it should be considered as resolved.
+		 */
+		result = (sqlstate == ERRCODE_UNDEFINED_OBJECT);
+	}
+	else
+		result = true;
+
+	elog(DEBUG1, "%s prepared foreign transaction on server %u with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 state->serverid,
+		 state->fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->am_participant_of_ac = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/*
+	 * Regardless of the event type, we can now mark ourselves as out of the
+	 * transaction.
+	 */
+	xact_got_connection = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
+
+static bool
+pgfdw_rollback_transaction(ConnCacheEntry *entry)
+{
+	bool abort_cleanup_failure = false;
+
+	/*
+	 * In rollback local transaction, if we don't the connection
+	 * it means any transaction started. So we can regard it as
+	 * success.
+	 */
+	if (!entry || !entry->conn)
+		return true;
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is already unsalvageable, do only the cleanup
+	 * and don't touch it further.
+	 */
+	if (entry->changing_xact_state)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return true;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+	else
+	{
+		entry->have_prep_stmt = false;
+		entry->have_error = false;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return !abort_cleanup_failure;
+}
+
+static bool
+pgfdw_commit_transaction(ConnCacheEntry *entry)
+{
+	PGresult	*res;
+	bool result = false;
+
+	if (!entry || !entry->conn)
+		return false;
+
+	/*
+	 * If abort cleanup previously failed for this connection,
+	 * we can't issue any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	/*
+	 * If there were any errors in subtransactions, and we
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+	entry->have_prep_stmt = false;
+	entry->have_error = false;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 21a2ef5..15dadf4 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,15 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -185,16 +207,19 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                                      List of foreign tables
- Schema |   Table    |  Server   |                   FDW options                    | Description 
---------+------------+-----------+--------------------------------------------------+-------------
- public | ft1        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft2        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft4        | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
- public | ft5        | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft6        | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft_pg_type | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
-(6 rows)
+                                         List of foreign tables
+ Schema |      Table       |  Server   |                   FDW options                    | Description 
+--------+------------------+-----------+--------------------------------------------------+-------------
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
+(9 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8650,3 +8675,345 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+\det+
+                                                 List of foreign tables
+ Schema |      Table       |  Server   |                            FDW options                            | Description 
+--------+------------------+-----------+-------------------------------------------------------------------+-------------
+ public | fpagg_tab_p1     | loopback  | (table_name 'pagg_tab_p1')                                        | 
+ public | fpagg_tab_p2     | loopback  | (table_name 'pagg_tab_p2')                                        | 
+ public | fpagg_tab_p3     | loopback  | (table_name 'pagg_tab_p3')                                        | 
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')                             | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1', use_remote_estimate 'true') | 
+ public | ft3              | loopback  | (table_name 'loct3', use_remote_estimate 'true')                  | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')                             | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type')                  | 
+ public | ftprt1_p1        | loopback  | (table_name 'fprt1_p1', use_remote_estimate 'true')               | 
+ public | ftprt1_p2        | loopback  | (table_name 'fprt1_p2')                                           | 
+ public | ftprt2_p1        | loopback  | (table_name 'fprt2_p1', use_remote_estimate 'true')               | 
+ public | ftprt2_p2        | loopback  | (table_name 'fprt2_p2', use_remote_estimate 'true')               | 
+ public | rem1             | loopback  | (table_name 'loc1')                                               | 
+ public | rem2             | loopback  | (table_name 'loc2')                                               | 
+(19 rows)
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+  srvname  
+-----------
+ loopback
+ loopback2
+ loopback3
+(3 rows)
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO on;
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO ft8_twophase VALUES(3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO "S 1"."T 6" VALUES (4);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  4
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(5);
+INSERT INTO "S 1"."T 6" VALUES (5);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  4
+(1 row)
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(8);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Fails, cannot commit the distributed transaction if 2PC-non-capable
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+ERROR:  cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Disables atomic commit, and success the same case as above.
+SET foreign_twophase_commit TO off;
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+(5 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+(5 rows)
+
+-- Enable atomic commit, again.
+SET foreign_twophase_commit TO on;
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(10);
+INSERT INTO ft8_twophase VALUES(10);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+-- Fails, cannot prepare the transaction if non-supporeted
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(11);
+INSERT INTO ft9_not_twophase VALUES(11);
+PREPARE TRANSACTION 'gx1'; -- error
+ERROR:  cannot prepare a transaction that modified remote tables
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
diff --git a/contrib/postgres_fdw/option.c b/contrib/postgres_fdw/option.c
index 6854f1b..1f45b1c 100644
--- a/contrib/postgres_fdw/option.c
+++ b/contrib/postgres_fdw/option.c
@@ -108,7 +108,8 @@ postgres_fdw_validator(PG_FUNCTION_ARGS)
 		 * Validate option value, when we can do so without any context.
 		 */
 		if (strcmp(def->defname, "use_remote_estimate") == 0 ||
-			strcmp(def->defname, "updatable") == 0)
+			strcmp(def->defname, "updatable") == 0 ||
+			strcmp(def->defname, "two_phase_commit") == 0)
 		{
 			/* these accept only boolean values */
 			(void) defGetBoolean(def);
@@ -177,6 +178,8 @@ InitPgFdwOptions(void)
 		/* fetch_size is available on both server and table */
 		{"fetch_size", ForeignServerRelationId, false},
 		{"fetch_size", ForeignTableRelationId, false},
+		/* two phase commit support */
+		{"two_phase_commit", ForeignServerRelationId, false},
 		{NULL, InvalidOid, false}
 	};
 
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index fd20aa9..1135046 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,8 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "catalog/pg_class.h"
@@ -359,6 +361,7 @@ static void postgresGetForeignUpperPaths(PlannerInfo *root,
 							 RelOptInfo *input_rel,
 							 RelOptInfo *output_rel,
 							 void *extra);
+static bool postgresIsTwoPhaseCommitEnabled(Oid serverid);
 
 /*
  * Helper functions
@@ -452,7 +455,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 				  const PgFdwRelationInfo *fpinfo_o,
 				  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -506,10 +508,29 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+	routine->ResolveForeignTransaction = postgresResolveForeignTransaction;
+	routine->IsTwoPhaseCommitEnabled = postgresIsTwoPhaseCommitEnabled;
+
 	PG_RETURN_POINTER(routine);
 }
 
 /*
+ * postgresIsTwoPhaseCommitEnabled
+ */
+static bool
+postgresIsTwoPhaseCommitEnabled(Oid serverid)
+{
+	ForeignServer	*server = GetForeignServer(serverid);
+
+
+	return server_uses_twophase_commit(server);
+}
+
+/*
  * postgresGetForeignRelSize
  *		Estimate # of rows and width of the result of the scan
  *
@@ -1356,7 +1377,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2411,7 +2432,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2704,7 +2725,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								&retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3321,7 +3342,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4108,7 +4129,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4198,7 +4219,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4421,7 +4442,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
@@ -5803,3 +5824,26 @@ find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel)
 	/* We didn't find any suitable equivalence class expression */
 	return NULL;
 }
+
+/*
+ * server_uses_twophase_commit
+ * Returns true if the foreign server is configured to support 2PC.
+ */
+bool
+server_uses_twophase_commit(ForeignServer *server)
+{
+	ListCell		*lc;
+
+	/* Check the options for two phase compliance */
+	foreach(lc, server->options)
+	{
+		DefElem    *d = (DefElem *) lfirst(lc);
+
+		if (strcmp(d->defname, "two_phase_commit") == 0)
+		{
+			return defGetBoolean(d);
+		}
+	}
+	/* By default a server is not 2PC compliant */
+	return false;
+}
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index 70b538e..f01a71a 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/relation.h"
@@ -115,7 +116,8 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
+extern PGconn *GetExistingConnection(Oid umid);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -123,6 +125,11 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 				   bool clear, const char *sql);
+extern bool postgresPrepareForeignTransaction(FdwXactResolveState *state);
+extern bool postgresCommitForeignTransaction(FdwXactResolveState *state);
+extern bool postgresRollbackForeignTransaction(FdwXactResolveState *state);
+extern bool postgresResolveForeignTransaction(FdwXactResolveState *state,
+											  bool is_commit);
 
 /* in option.c */
 extern int ExtractConnectionOptions(List *defelems,
@@ -181,6 +188,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 						List *remote_conds, List *pathkeys, bool is_subquery,
 						List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 88c4cb4..2554c9c 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,19 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -2304,7 +2331,6 @@ SELECT t1.a, t2.b FROM fprt1 t1 INNER JOIN fprt2 t2 ON (t1.a = t2.b) WHERE t1.a
 
 RESET enable_partitionwise_join;
 
-
 -- ===================================================================
 -- test partitionwise aggregates
 -- ===================================================================
@@ -2354,3 +2380,126 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+
+\det+
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO on;
+
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO ft8_twophase VALUES(3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO "S 1"."T 6" VALUES (4);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(5);
+INSERT INTO "S 1"."T 6" VALUES (5);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(8);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Fails, cannot commit the distributed transaction if 2PC-non-capable
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Disables atomic commit, and success the same case as above.
+SET foreign_twophase_commit TO off;
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Enable atomic commit, again.
+SET foreign_twophase_commit TO on;
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(10);
+INSERT INTO ft8_twophase VALUES(10);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Fails, cannot prepare the transaction if non-supporeted
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(11);
+INSERT INTO ft9_not_twophase VALUES(11);
+PREPARE TRANSACTION 'gx1'; -- error
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 54b5e98..f4a9ff5 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -436,6 +436,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
-- 
2.10.5

From 38b3181eb9dd8f2963132017888355da8d3f6a64 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:48:08 +0900
Subject: [PATCH v19 4/4] Add regression tests for atomic commit.

---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index daf79a0..71c8b9d 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000..a23f120
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port', two_phase_commit 'on');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port', two_phase_commit 'on');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_prepared_fdw_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 6890678..d1b181a 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2286,9 +2286,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2303,7 +2306,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.10.5

From ac0bf1880847507317a6d7b871fae4f28f783048 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:44:42 +0900
Subject: [PATCH v19 2/4] Support atomic commit among multiple foreign servers.

---
 doc/src/sgml/catalogs.sgml                    |   97 +
 doc/src/sgml/config.sgml                      |  124 ++
 doc/src/sgml/distributed-transaction.sgml     |  152 ++
 doc/src/sgml/fdwhandler.sgml                  |  192 ++
 doc/src/sgml/filelist.sgml                    |    1 +
 doc/src/sgml/func.sgml                        |   51 +
 doc/src/sgml/monitoring.sgml                  |   56 +
 doc/src/sgml/postgres.sgml                    |    1 +
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/TAGS               |    1 +
 src/backend/access/fdwxact/fdwxact.c          | 2610 +++++++++++++++++++++++++
 src/backend/access/fdwxact/fdwxact_launcher.c |  641 ++++++
 src/backend/access/fdwxact/fdwxact_resolver.c |  331 ++++
 src/backend/access/rmgrdesc/Makefile          |    8 +-
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   65 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/Makefile           |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   32 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/foreigncmds.c            |   23 +
 src/backend/executor/execPartition.c          |    4 +
 src/backend/executor/nodeForeignscan.c        |    8 +
 src/backend/executor/nodeModifyTable.c        |    5 +
 src/backend/foreign/foreign.c                 |   43 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   18 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    2 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   61 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  150 ++
 src/include/access/fdwxact_launcher.h         |   32 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   52 +
 src/include/access/resolver_internal.h        |   67 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   23 +
 src/include/foreign/fdwapi.h                  |   18 +-
 src/include/foreign/foreign.h                 |    2 +-
 src/include/pgstat.h                          |    8 +-
 src/include/storage/proc.h                    |   10 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    2 +
 src/test/regress/expected/rules.out           |   12 +
 62 files changed, 5147 insertions(+), 27 deletions(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 120000 src/backend/access/fdwxact/TAGS
 create mode 100755 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/fdwxact_launcher.c
 create mode 100644 src/backend/access/fdwxact/fdwxact_resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 6d6fbec..9d99cdc 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9622,6 +9622,103 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-prepared-fdw-xacts">
+  <title><structname>pg_prepared_fdw_xacts</structname></title>
+
+  <indexterm zone="view-pg-prepared-fdw-xacts">
+   <primary>pg_prepared_fdw_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_prepared_fdw_xacts</structname> displays
+   information about foreign transactions that are currently prepared on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="fdw-transaction-managements"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_prepared_xacts</structname> contains one row per prepared
+   foreign transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_prepared_fdw_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>transaction</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Transaction id that this foreign transaction associates with
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server that this foreign server is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction: <literal>prepared</literal>, <literal>committing</literal>, <literal>aborting</literal> or <literal>unknown</literal>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7554cba..589ef6e 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -1547,6 +1547,29 @@ include_dir 'conf.d'
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+      <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Sets the maximum number of foreign transactions that can be prepared
+        simultaneously. A single local transaction can give rise to multiple
+        foreign transaction. If <literal>N</literal> local transactions each
+        across <literal>K</literal> foreign server this value need to be set
+        <literal>N * K</literal>, not just <literal>N</literal>.
+        This parameter can only be set at server start.
+       </para>
+       <para>
+        When running a standby server, you must set this parameter to the
+        same or higher value than on the master server. Otherwise, queries
+        will not be allowed in the standby server.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry id="guc-work-mem" xreflabel="work_mem">
       <term><varname>work_mem</varname> (<type>integer</type>)
       <indexterm>
@@ -3612,6 +3635,78 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
      </variablelist>
     </sect2>
 
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+
+     <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+      <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+      <indexterm>
+       <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+        resolver is responsible for foreign transaction resolution on one database.
+       </para>
+       <para>
+        Foreign transaction resolution workers are taken from the pool defined by
+        <varname>max_worker_processes</varname>.
+       </para>
+       <para>
+        The default value is 0.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry id="foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+      <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specify how long the foreign transaction resolver should wait when the last resolution
+        fails before retrying to resolve foreign transaction. This parameter can only be set in the
+        <filename>postgresql.conf</filename> file or on the server command line.
+       </para>
+       <para>
+        The default value is 10 seconds.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry id="foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+      <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+      <indexterm>
+       <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+      </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Terminate foreign transaction resolver processes that don't have any foreign
+        transactions to resolve longer than the specified number of milliseconds.
+        A value of zero disables the timeout mechanism.  You should set this value to
+        zero only if you set <varname>max_foreign_transaction_resolvers</varname> as
+        much as databases you have. This parameter can only be set in the
+        <filename>postgresql.conf</filename> file or on the server command line.
+       </para>
+       <para>
+        The default value is 60 seconds.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     </variablelist>
+    </sect2>
+
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -7827,6 +7922,35 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-foreign-transaction">
+    <title>Foreign Transaction Management</title>
+
+    <variablelist>
+
+     <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophase_commit">
+      <term><varname>foreign_twophase_commit</varname> (<type>bool</type>)
+       <indexterm>
+        <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+       </indexterm>
+      </term>
+      <listitem>
+       <para>
+        Specifies whether transaction commit will wait for all involving foreign transaction
+        to be resolved before the command returns a "success" indication to the client.
+        Both <varname>max_prepared_foreign_transactions</varname> and
+        <varname>max_foreign_transaction_resolvers</varname> must be non-zero value to
+        allow foreign twophase commit to be used.
+       </para>
+       <para>
+        This parameter can be changed at any time; the behavior for any one transaction
+        is determined by the setting in effect when it commits.
+       </para>
+      </listitem>
+     </varlistentry>
+    </variablelist>
+
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000..54e582e
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,152 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction Management</title>
+
+ <para>
+  This chapter explains what distributed transaction management is, and how it can be configured
+  in PostgreSQL.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit is an operation that applies a set of changes as a single operation
+   globally. <productname>PostgreSQL</productname> provides a way to perform a transaction
+   with foreign resources using <literal>Foreign Data Wrapper</literal>. Using the
+   <productname>PostgreSQL</productname>'s atomic commit ensures that all changes
+   on foreign servers end in either commit or rollback using the transaction callback
+   routines (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employees Two-phase commit protocol, which is a
+    type of atomic commitment protocol (ACP). Using Two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed transaction manager
+    prepares all transaction on the foreign servers if two-phase commit is required.
+    Two-phase commit is required only if the transaction modifies data on two or more
+    servers including the local server itself and user requests it by
+    <xref linkend="guc-foreign-twophase-commit"/>. If all preparations on foreign servers
+    got successful go to the next step. Any faliure happens in this step
+    <productname>PostgreSQL</productname> changes over rollback, then rollback all transactions
+    on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the transaction
+    locally. Any failure happens in this step <productname>PostgreSQL</productname> changes
+    over rollback, then rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign Transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Foreign Transaction Resolution</title>
+
+   <para>
+    Foreign transaction resolutions are performed by foreign transaction resolver process.
+    They commit all prepared transaction on foreign servers if the coordinator received
+    an agreement message from all foreign server during the first step. On the other hand,
+    if any foreign server failed to prepare the transaction, it rollbacks all prepared
+    transactions.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions on one
+    database of the coordinator side. On failure during resolution, they retries to
+    resolve after <varname>foreign_transaction_resolution_interval</varname>.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>In-doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit or rollback
+    using two-phase commit protocol. However, if the second phase fails for whatever reason
+    the transaction becomes in-doubt. The transactions becomes in-doubt in the following
+    situations:
+
+   <itemizedlist>
+    <listitem>
+     <para>
+      A local <productname>PostgreSQL</productname> server crashes during atomic commit
+      operation.
+     </para>
+    </listitem>
+
+    <listitem>
+     <para>
+      A local <productname>PostgreSQL</productname> server got a cancellation by user during
+      atomic commit.
+     </para>
+    </listitem>
+   </itemizedlist>
+
+   In-doubt transactions are automatically handled by foreign transaction resolver process
+   when there is no online transaction requesting resolutions.
+   <function>pg_resolve_fdw_xact</function> provides a way to resolve transactions on foreign
+   servers that are participate the distributed transaction manually.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-fdwxact-resolver-view"><literal>pg_stat_fdwxact_resolver</literal></link>
+    view. This view contains one row for every foreign Transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 4ce88dd..5180ce0 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1390,6 +1390,103 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     If an FDW wishes to support <firstterm>atomic commit</firstterm>
+     (as described in <xref linkend="fdw-transaction-managements"/>), it must call the
+     registrasaction function <function>FdwXactRegisterForeignTransaction</function>
+     and provide the following callback functions:
+    </para>
+
+    <para>
+<programlisting>
+bool
+PrepareForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if atomic commit is required.
+    Returning <literal>true</literal> means that preparing the foreign
+    transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Commit the not-prepared transaction on the foreign server.
+    This function is called at the pre-commit phase of local
+    transaction if atomic commit is not required. The atomic
+    commit is not required either when we modified data on
+    only one server including the local server or when userdoesn't
+    request atomic commit by <xref linkend="guc-foreign-twophase-commit"/>.
+    Returning <literal>true</literal> means that commit the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Rollback a not-prepared transaction on the foreign server.
+    This function is called at the end of local transaction after
+    rollbacked locally either when user requested rollback or when
+    any error occurs during the transaction. This function could
+    be called recursively if any error occurs during rollback the
+    foreign transaction for whatever reason. You need to track
+    recursion and prevent this function from being called infinitely.
+    Returning <literal>true</literal> means that rollback the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+ResolvePreparedForeignTransaction(FdwXactResolveState *state,
+                                  bool is_commit);
+</programlisting>
+    Commit or rollback the prepared transaction on the foreign server.
+    When <varname>is_commit</varname> is true, it indicates that the foreign
+    transaction should be committed. Otherwise the foreign transaction should
+    be aborted.
+    This function normally is called by the foreign transaction resolver
+    process but can also be called by <function>pg_resovle_fdw_xacts</function>
+    function. In the resolver process, this function is called either
+    when a backend requests the resolver process to resolve a distributed
+    transaction after prepared, or when a database has dangling
+    transactions. Returning <literal>true</literal> means that resolving
+    the foreign transaction got successful.
+    In abort case, please note that the prepared transaction identified
+    by <varname>state->fdwxact_id</varname> might not exist on the foreign
+    server. If you failed to resolve the foreign transaction due to undefined
+    object error (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) you should
+   regards it as success and return <literal>true</literal>.
+    </para>
+    <para>
+<programlisting>
+bool
+IsTwoPhaseCommitEnabled(Oid serverid);
+</programlisting>
+    Return <literal>true</literal> if the foreign server identified by
+    <literal>serverid</literal> is capable of two-phase commit protocol.
+    This function is called when the transaction begins to modify data on
+    the foreign server. Return <literal>false</literal> indicates that
+    the current transaction cannot use atomic commit even if atomic commit
+    is requested by user.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1835,4 +1932,99 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+    <title>Transaction managements for Foreign Data Wrappers</title>
+
+    <para>
+     <productname>PostgreSQL</productname> foreign transaction manager
+     allows FDWs to read and write data on foreign server within a transaction while
+     maintaining atomicity of the foreign data (aka atomic commit). Using
+     atomic commit, it guarantees that a distributed transaction is committed
+     or rollbacked on all participants foreign
+     server.  To achieve atomic commit, <productname>PostgreSQL</productname>
+     employees two-phase commit protocol, which is a type of atomic commitment
+     protocol. Every FDW that wish to support atomic commit
+     is required to support the transaction management callback routines
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details)
+     and register the foreign transaction using
+     <function>FdwXactRegisterForeignTransaction</function> when starting a
+     transaction on the foreign server. Transaction of registered foreign server
+     is managed by the foreign transaction manager.
+<programlisting>
+void
+FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, char *fx_id)
+</programlisting>
+    This function should be called when a transaction starts on the foreign server.
+    <varname>serverid</varname> and <varname>userid</varname> are <type>OID</type>s
+    which specify the transaction starts on what server by who. <varname>fx_id</varname>
+    is null-terminated string which is an identifer of foreign transaction and it
+    will be passed when transaction management APIs is called. The length of
+    <varname>fx_id</varname> must be less than 200 bytes. Also this identifier
+    must be unique enough so that it doesn't conflict other concurrent foreign
+    transactions. <varname>fx_id</varname> can be <literal>NULL</literal>.
+    If it's <literal>NULL</literal>, a unique transaction identifier is automacitally
+    generated with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    Since this identifier is used per foreign transaction and the xid of unresolved
+    distributed transaction never reused, an auto-generated identifier is fairly
+    enough to ensure uniqueness. It's recommended to generate foreign transaction
+    identifier in FDW if the format of auto-generated identifier doesn't match
+    the requirement of the foreign server.
+    </para>
+
+    <para>
+     An example of such transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When a transaction starts on the foreign server, FDW that wishes atomic
+     commit must register the foreign transaction as a participant by calling
+     <function>FdwXactRegisterForeignTransaction</function>. Also during
+     transaction, <function>IsTwoPhaseCommitEnabled</function> is called whenever
+     the transaction begins to modify data on the foreign server. If FDW wishes
+     atomic commit <function>IsTwoPhaseCommitEnabled</function> must return
+     <literal>true</literal>. All foreign transaction participants must
+     return <literal>true</literal> to achieve atomic commit.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling <function>PrepareForeignTransaction</function>
+     if two-phase commit protocol is required. Two-phase commit is required only if
+     the transaction modified data on more than one servers including the local
+     server and user requests atomic commit. <productname>PostgreSQL</productname>
+     can commit locally and go to the next step if and only if all preparing foreign
+     transactions got successful. If two-phase commit is not required, the foreign
+     transaction manager commits the transaction on the foreign server by calling
+     <function>CommitForeignTransaction</function> and then
+     <productname>PostgreSQL</productname> commits locally. The foreign transaction
+     manager doesn't do any further change on foreign transactions from this point
+     forward. If any failure happens during the transaction for whatever reason,
+     for example a network failure or user request until
+     <productname>PostgreSQL</productname> commits locally the foreign transaction
+     manager changes over to rollback and calls
+     <function>RollbackForeignTransaction</function> for every foreign servers to
+     close the current transaction on foreign servers.
+    </para>
+
+    <para>
+     When two-phase commit is required, after committed locally, the transaction
+     commits will wait for all prepared foreign transaction to be resolved before
+     the commit completes. One foreign transaction resolver is responsible for
+     foreign transaction resolution on a database.
+     <function>ResolverForeignTransaction</function> is called by the foreign
+     transaction resolver process when resolution.
+     <function>ResolveForeignTransaction</function> is also be called
+     when user executes <function>pg_resovle_fdw_xact</function> function.
+    </para>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 48ac14a..38d6fcb 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -49,6 +49,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 5193df3..8bb251e 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -20806,6 +20806,57 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-fdw-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_fdw_xacts</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_fdw_xacts</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function search for foreign transaction
+        matching the arguments and resolves then. This function won't resolve
+        a foreign transaction which is in progress, or one that is locked by some
+        other backend.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_fdw_xact</function>
+        except it remove foreign transaction entry without resolving.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 0484cfa..6b2aa6f 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -332,6 +332,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_fdw_xact_resolver</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-fdwxact-resolver-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1194,6 +1202,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
          <entry><literal>LogicalLauncherMain</literal></entry>
          <entry>Waiting in main loop of logical launcher process.</entry>
         </row>
@@ -1405,6 +1421,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
+        <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
          <entry>Waiting during base backup when throttling activity.</entry>
@@ -2214,6 +2234,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-fdwxact-resolver-view" xreflabel="pg_stat_fdw_xact_resolver">
+   <title><structname>pg_stat_fdw_xact_resolver</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index 0070603..c10e21f 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -164,6 +164,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index bd93a6a..4a1ebdc 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  tablesample transam
+			  tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000..9ddbb14
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o fdwxact_resolver.o fdwxact_launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/TAGS b/src/backend/access/fdwxact/TAGS
new file mode 120000
index 0000000..1a96393
--- /dev/null
+++ b/src/backend/access/fdwxact/TAGS
@@ -0,0 +1 @@
+/home/masahiko/source/postgresql/TAGS
\ No newline at end of file
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100755
index 0000000..f57ad8e
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2610 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL distributed transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers automically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * When a foreign data wrapper starts transaction on a foreign server that
+ * is capable of two-phase commit protocol, foreign data wrappers registers
+ * the foreign transaction using function FdwXactRegisterForeignTransaction()
+ * in order to participate to a group for atomic commit. Participants are
+ * identified by oid of foreign server and user. When the foreign transaction
+ * begins to modify data the executor marks it as modified using
+ * FdwXactMarkForeignTransactionModified().
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. After committing or rolling back locally, we
+ * notify the resolver process and tell it to commit or roll back those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail when an ERROR
+ * after already committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * waiters each time we receive a request. We have two queues: the active
+ * queue and the retry queue. The backend is inserted to the active queue at
+ * first, and then it is moved to the retry queue by the resolver process if
+ * the resolution fails. The backends in the retry queue are processed at
+ * interval of foreign_transaction_resolution_retry_interval.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or more
+ * servers including itself. In other case, all foreign transactions are
+ * committed during pre-commit.
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. dangling
+ * transaction). Dangling transactions are processed by the resolve process
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * 	* On PREPARE redo we add the foreign transaction to FdwXactCtl->fdw_xacts.
+ *	  We set fdw_xact->inredo to true for such entries.
+ *	* On Checkpoint redo, we iterate through FdwXactCtl->fdw_xacts entries that
+ *	  have set fdw_xact->inredo true and are behind the redo_horizon. We save
+ *    them to disk and then set fdw_xact->ondisk to true.
+ *	* On COMMIT and ABORT we delete the entry from FdwXactCtl->fdw_xacts.
+ *	  If fdw_xact->ondisk is true, we delete the corresponding file from
+ *	  the disk as well.
+ *  * RecoverFdwXacts loads all foreign transaction entries from disk into
+ *    memory at server startup.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Is atomic commit requested by user? */
+#define AtomicCommitRequested() \
+	(foreign_twophase_commit == true && \
+	 max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Structure to bundle the foreign transaction participant */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in global entry. NULL if
+	 * this foreign transaction is registered but not inserted
+	 * yet.
+	 */
+	FdwXact		fdw_xact;
+	char		*fdw_xact_id;
+
+	/* Participant server and its user mapping */
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	/* true if this transaction modified data on the foreign server */
+	bool		modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact;
+	CommitForeignTransaction_function	commit_foreign_xact;
+	RollbackForeignTransaction_function	rollback_foreign_xact;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit.
+ * This list has only foreign servers that are capable of two-phase
+ * commit protocol.
+ *
+ * We can manage all foreign transactions involving with the transaction
+ * regardless of their configuration but we didn't because it requires all
+ * FDWs to register foreign server anyway, which breaks the backward
+ * compatibility.
+ */
+List *FdwXactParticipantsForAC = NIL;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDW_XACTS_DIR "pg_fdw_xact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDW_XACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDW_XACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+static FdwXact FdwXactRegisterFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static bool FdwXactResolveForeignTransaction(FdwXactResolveState *state, FdwXact fdwxact,
+											 int elevel);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactQueueInsert(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid, bool give_warnings);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							  bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid, Oid serverid,
+								  Oid userid, XLogRecPtr insert_start_lsn,
+								  bool give_warnings);
+static List *get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						   bool need_lock);
+static FdwXact get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								bool need_lock);
+static FdwXact get_all_fdw_xacts(int *length);
+static FdwXact insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							   Oid umid, char *fdw_xact_id);
+static char *generate_fdw_xact_identifier(Oid serverid, Oid userid);
+static void remove_fdw_xact(FdwXact fdw_xact);
+static FdwXactResolveState *create_fdw_xact_resovle_state(void);
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+bool foreign_twophase_commit = false;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * This function aimes to be called by FDW when foreign transaction
+ * starts. The foreign server identified by given server id must
+ * support atomic commit APIs. The foreign transaction is identified
+ * by given identifier 'fdwxact_id' which can be NULL. If it's NULL,
+ * we construct an unique identifer instead.
+ *
+ * Registered foreign transaction are managed by foreign transaction
+ * manager until the end of the transaction.
+ */
+void
+FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, char *fdwxact_id)
+{
+	FdwXactParticipant	*fdw_part;
+	ListCell   			*lc;
+	ForeignServer 		*foreign_server;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	FdwRoutine			*fdw_routine;
+	MemoryContext		old_ctx;
+
+	/* Check length of foreign transaction identifier */
+	if (fdwxact_id != NULL && strlen(fdwxact_id) >= NAMEDATALEN)
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifer \"%s\" is too long",
+						fdwxact_id),
+				 errdetail("foreign transaction identifier must be less than %d characters.",
+						   NAMEDATALEN)));
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	/* Duplication check */
+	foreach(lc, FdwXactParticipantsForAC)
+	{
+		fdw_part = lfirst(lc);
+
+		/* Quick return if there is already registered connection */
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+			ereport(ERROR,
+					(errmsg("attempt to start transction again on server %u user %u",
+							serverid, userid)));
+	}
+
+	/*
+	 * Participants information is needed at the end of a transaction, when
+	 * system cache are not available. so save it in TopTransactionContext
+	 * beforehand so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	foreign_server = GetForeignServer(serverid);
+	fdw = GetForeignDataWrapper(foreign_server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+	user_mapping = GetUserMapping(userid, serverid);
+
+	/* Make sure that the FDW has transaction handlers */
+	if (!fdw_routine->PrepareForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function provided for preparing foreign transaction for FDW %s",
+						fdw->fdwname)));
+	if (!fdw_routine->CommitForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to commit a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+	if (!fdw_routine->RollbackForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to rollback a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+
+	/* Generate unique identifier if not provided */
+	if (fdwxact_id ==  NULL)
+		fdwxact_id = generate_fdw_xact_identifier(serverid, userid);
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdw_xact_id = fdwxact_id;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdw_xact = NULL;
+	fdw_part->modified = false;	/* by default */
+	fdw_part->prepare_foreign_xact = fdw_routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact = fdw_routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact = fdw_routine->RollbackForeignTransaction;
+
+	/* Add this foreign transaction to the participants list */
+	FdwXactParticipantsForAC = lappend(FdwXactParticipantsForAC, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+
+	return;
+}
+
+/*
+ * Remember the registered foreign transaction modified data. This function
+ * is called by the executor when it begins to modify data on a foreign server
+ * regardless the foreign server is capable of two-phase commit protocol.
+ * Marking it will be used to determine we must use two-phase commit protocol
+ * at commit. This function also checks if the begin modified foreign server
+ * is capable of two-phase commit or not. If it doesn't support, we remember
+ * it in MyXactFlags.
+ */
+void
+FdwXactMarkForeignTransactionModified(ResultRelInfo *resultRelInfo, int flags)
+{
+	Relation			rel = resultRelInfo->ri_RelationDesc;
+	FdwXactParticipant	*fdw_part;
+	ForeignTable		*ftable;
+	ListCell   			*lc;
+	Oid					userid;
+	Oid					serverid;
+
+	bool found = false;
+
+	/* Quick return if user not request */
+	if (!AtomicCommitRequested())
+		return;
+
+	/* Do nothing in EXPLAIN (no ANALYZE) case */
+	if (flags && EXEC_FLAG_EXPLAIN_ONLY)
+		return;
+
+	ftable = GetForeignTable(RelationGetRelid(rel));
+
+	/*
+	 * If the being modified foreign server doesn't or cannot enable
+	 * two-phase commit protocol, mark that we've written such server
+	 * and return.
+	 */
+	if (resultRelInfo->ri_FdwRoutine->IsTwoPhaseCommitEnabled == NULL ||
+		!resultRelInfo->ri_FdwRoutine->IsTwoPhaseCommitEnabled(ftable->serverid))
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		return;
+	}
+
+	/*
+	 * The foreign server being modified supports two-phase commit protocol,
+	 * remember that the foreign transaction modified data.
+	 */
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	serverid = ftable->serverid;
+	foreach(lc, FdwXactParticipantsForAC)
+	{
+		fdw_part = lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			fdw_part->modified = true;
+			found = true;
+			break;
+		}
+	}
+
+	if (!found)
+		elog(ERROR, "attempt to mark unregistered foreign server %u, user %u as modified",
+			 serverid, userid);
+}
+
+/*
+ * FdwXactShmemSize
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdw_xacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * FdwXactShmemInit
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdw_xacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->freeFdwXacts = NULL;
+		FdwXactCtl->numFdwXacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdw_xacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdw_xacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdw_xacts[cnt].status = FDW_XACT_INITIAL;
+			fdw_xacts[cnt].fxact_free_next = FdwXactCtl->freeFdwXacts;
+			FdwXactCtl->freeFdwXacts = &fdw_xacts[cnt];
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * PreCommit_FdwXacts
+ *
+ * This function prepares all foreign transaction participants if atomic commit
+ * is required. Otherwise commits them without preparing.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipantsForAC == NIL)
+		return;
+
+	/*
+	 * If user requires the atomic commit semantics, we don't allow COMMIT if we've
+	 * modified data on foreign servers both that can execute two-phase commit
+	 * protocol and that cannot.
+	 */
+	if (foreign_twophase_commit == true &&
+		((MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0) )
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	if (ForeignTwophaseCommitRequired())
+	{
+		/*
+		 * Prepare the transactions on the all foreign servers. If any prepare
+		 * transaction fails for whatever reason, we change over aborts.
+		 */
+		FdwXactPrepareForeignTransactions();
+
+		/* keep FdwXactparticipantsForAC until end of transaction */
+	}
+	else
+	{
+		ListCell   *lc;
+
+		/* Two-phase commit is not required, commit them */
+		foreach(lc, FdwXactParticipantsForAC)
+		{
+			FdwXactResolveState *state;
+			FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+			state = create_fdw_xact_resovle_state();
+			state->serverid = fdw_part->server->serverid;
+			state->userid = fdw_part->usermapping->userid;
+			state->umid = fdw_part->usermapping->umid;
+
+			/* Commit foreign transaction */
+			if (!fdw_part->commit_foreign_xact(state))
+				ereport(ERROR,
+						(errmsg("could not commit foreign transaction on server %s",
+								fdw_part->server->servername)));
+		}
+
+		/* Forget all participants */
+		FdwXactParticipantsForAC = NIL;
+	}
+}
+
+/*
+ * FdwXactPrepareForeignTransactions
+ *
+ * Prepare all foreign transaction participants.  This function creates a prepared
+ * participants chain each time when we prepared a foreign transaction. The prepared
+ * participants chain is used to access all participants of distributed transaction
+ * quickly. If any one of them fails to prepare, we change over aborts.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	FdwXactResolveState *state;
+	ListCell   *lcell;
+	FdwXact		prev_fdwxact = NULL;
+
+	state = create_fdw_xact_resovle_state();
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactParticipantsForAC)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXact		fdwxact;
+
+		/*
+		 * Register the foreign transaction entry. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fdwxact = FdwXactRegisterFdwXactEntry(GetTopTransactionId(), fdw_part);
+
+		state->serverid = fdw_part->server->serverid;
+		state->userid = fdw_part->usermapping->userid;
+		state->umid = fdw_part->usermapping->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		/*
+		 * Between FdwXactRegisterFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal). During abort processing,
+		 * we might try to resolve a never-prepared transaction, and get an error.
+		 * This is fine as long as the FDW provides us unique prepared transaction
+		 * identifiers.
+		 */
+		if (!fdw_part->prepare_foreign_xact(state))
+		{
+			/* Failed to prepare, change over aborts */
+			ereport(ERROR,
+					(errmsg("could not prepare transaction on foreign server %s",
+							fdw_part->server->servername)));
+		}
+
+		/* Preparation is success, update its status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdw_part->fdw_xact->status = FDW_XACT_PREPARED;
+		fdw_part->fdw_xact = fdwxact;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Create a prepared participants chain, which is link-ed FdwXact entries
+		 * involving with this transaction.
+		 */
+		if (prev_fdwxact)
+		{
+			/* Append others to the tail */
+			Assert(fdwxact->fxact_next == NULL);
+			prev_fdwxact->fxact_next = fdwxact;
+		}
+	}
+}
+
+/*
+ * FdwXactRegisterFdwXactEntry
+ *
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and will
+ * be persisted to the disk under pg_fdw_xact directory when checkpoint.
+ */
+static FdwXact
+FdwXactRegisterFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fxact;
+	FdwXactOnDiskData	*fxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact = insert_fdw_xact(MyDatabaseId, xid, fdw_part->server->serverid,
+							fdw_part->usermapping->userid,
+							fdw_part->usermapping->umid, fdw_part->fdw_xact_id);
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->held_by = MyBackendId;
+	fdw_part->fdw_xact = fxact;
+	LWLockRelease(FdwXactLock);
+
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdw_xact_id);
+	data_len = data_len + strlen(fdw_part->fdw_xact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fxact_file_data->dbid = MyDatabaseId;
+	fxact_file_data->local_xid = xid;
+	fxact_file_data->serverid = fdw_part->server->serverid;
+	fxact_file_data->userid = fdw_part->usermapping->userid;
+	fxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fxact_file_data->fdw_xact_id, fdw_part->fdw_xact_id,
+		   strlen(fdw_part->fdw_xact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fxact_file_data, data_len);
+	fxact->insert_end_lsn = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_INSERT);
+	XLogFlush(fxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact->valid = true;
+	LWLockRelease(FdwXactLock);
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fxact_file_data);
+	return fxact;
+}
+
+/*
+ * insert_fdw_xact
+ *
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				Oid umid, char *fdw_xact_id)
+{
+	int i;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		fxact = FdwXactCtl->fdw_xacts[i];
+		if (fxact->dbid == dbid &&
+			fxact->local_xid == xid &&
+			fxact->serverid == serverid &&
+			fxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->freeFdwXacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fxact = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fxact->fxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->numFdwXacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts++] = fxact;
+
+	fxact->held_by = InvalidBackendId;
+	fxact->dbid = dbid;
+	fxact->local_xid = xid;
+	fxact->serverid = serverid;
+	fxact->userid = userid;
+	fxact->umid = umid;
+	fxact->insert_start_lsn = InvalidXLogRecPtr;
+	fxact->insert_end_lsn = InvalidXLogRecPtr;
+	fxact->valid = false;
+	fxact->ondisk = false;
+	fxact->inredo = false;
+	memcpy(fxact->fdw_xact_id, fdw_xact_id, strlen(fdw_xact_id) + 1);
+
+	return fxact;
+}
+
+/*
+ * remove_fdw_xact
+ *
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdw_xact(FdwXact fdw_xact)
+{
+	int			cnt;
+
+	Assert(fdw_xact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Search the slot where this entry resided */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		if (FdwXactCtl->fdw_xacts[cnt] == fdw_xact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (cnt >= FdwXactCtl->numFdwXacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdw_xact->local_xid, fdw_xact->serverid, fdw_xact->userid)));
+
+	/* Remove the entry from active array */
+	FdwXactCtl->numFdwXacts--;
+	FdwXactCtl->fdw_xacts[cnt] = FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts];
+
+	/* Put it back into free list */
+	fdw_xact->fxact_free_next = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fdw_xact;
+
+	/* Reset informations */
+	fdw_xact->status = FDW_XACT_INITIAL;
+	fdw_xact->held_by = InvalidBackendId;
+	fdw_xact->fxact_next = NULL;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdw_xact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdw_xact->serverid;
+		record.dbid = fdw_xact->dbid;
+		record.xid = fdw_xact->local_xid;
+		record.userid = fdw_xact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdw_xact_remove));
+		recptr = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true if the current transaction requires foreign two-phase commit
+ * to achieve atomic commit. Foreign two-phase commit is required if we
+ * satisfy either case: we modified data on two or more foreign server, or
+ * we modified both non-temporary relation on local and data on more than
+ * one foreign server.
+ */
+bool
+ForeignTwophaseCommitRequired(void)
+{
+	int	nserverswritten = list_length(FdwXactParticipantsForAC);
+	ListCell*	lc;
+	bool		modified = false;
+
+	/* Return if not requested */
+	if (!AtomicCommitRequested())
+		return false;
+
+	/* Check if we modified data on any foreign server */
+	foreach(lc, FdwXactParticipantsForAC)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+		{
+			modified = true;
+			break;
+		}
+	}
+
+	/* We didn't modify data on any foreign server */
+	if (!modified)
+		return false;
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	return nserverswritten > 1;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * ForgetAllFdwXactParticipants
+ *
+ * Reset all the foreign transaction entries that this backend registered.
+ * If the foreign transaction has the corresponding FdwXact entry, resetting
+ * the held_by field means to leave that entry in unresolved state. If we
+ * leaves any entries, we update the oldest xmin of unresolved transaction
+ * so that transaction status of dangling transaction are not truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_lefts = 0;
+
+	if (FdwXactParticipantsForAC == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(cell, FdwXactParticipantsForAC)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+
+		/* Skip if didn't register FdwXact entry yet */
+		if (fdw_part->fdw_xact == NULL)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in
+		 * FdwXactParticipantsForAC could be used by other backend before we
+		 * forget in case where the resolver process removes the FdwXact entry
+		 * and other backend reuses it before we forget. So we need to check
+		 * if the entries are still associated with the transaction.
+		 */
+		if (fdw_part->fdw_xact->held_by == MyBackendId)
+		{
+			fdw_part->fdw_xact->held_by = InvalidBackendId;
+			n_lefts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Update the oldest local transaction of unresolved distributed
+	 * transaction if we leaved any FdwXact entries.
+	 */
+	if (n_lefts > 0)
+		FdwXactComputeRequiredXmin();
+
+	FdwXactParticipantsForAC = NIL;
+}
+
+/*
+ * AtProcExit_FdwXact
+ *
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+/*
+ * Wait for foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDW_XACT_NOT_WAITING and then change
+ * that state to FDW_XACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDW_XACT_WAIT_COMPLETE once foreign transactions are resolved.
+ * This backend then resets its state to FDW_XACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue and changes the state to FDW_XACT_WAITING_RETRY.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+	ListCell	*lc;
+	List		*fdwxact_participants = NIL;
+
+	/* Quick exit if atomic commit is not requested */
+	if (!AtomicCommitRequested())
+		return;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDW_XACT_NOT_WAITING);
+
+	if (FdwXactParticipantsForAC != NIL)
+	{
+		/*
+		 * If we're waiting for foreign transactions to be resolved that
+		 * we've prepared just before, use the participants list.
+		 */
+		Assert(MyPgXact->xid == wait_xid);
+		fdwxact_participants = FdwXactParticipantsForAC;
+	}
+	else
+	{
+		/*
+		 * Get participants list from the global array. This is required (1)
+		 * when we're waiting for foreign transactions to be resolved that
+		 * is part of a local prepared transaction that is marked as prepared
+		 * during running, or (2) when we resolve the PREPARE'd distributed
+		 * transaction after restart.
+		 */
+		fdwxact_participants = get_fdw_xacts(MyDatabaseId, wait_xid,
+											 InvalidOid, InvalidOid, true);
+	}
+
+	/* Exit if we found no foreign transaction to resolve */
+	if (fdwxact_participants == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(lc, fdwxact_participants)
+	{
+		FdwXact fdw_xact = (FdwXact) lfirst(lc);
+
+		/* Don't overwrite status if fate has been determined */
+		if (fdw_xact->status == FDW_XACT_PREPARED)
+			fdw_xact->status = (is_commit ?
+								FDW_XACT_COMMITTING_PREPARED :
+								FDW_XACT_ABORTING_PREPARED);
+	}
+
+	/* Set backend status and enqueue itself to the active queue*/
+	MyProc->fdwXactState = FDW_XACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	FdwXactQueueInsert();
+	LWLockRelease(FdwXactLock);
+
+	/* Launch a resolver process if not yet, or wake it up */
+	fdwxact_maybe_launch_resolver(false);
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDW_XACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDW_XACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDW_XACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+
+	/*
+	 * Forget the list of locked entries, also means that the entries
+	 * that could not resolved are remained as dangling transactions.
+	 */
+	ForgetAllFdwXactParticipants();
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Acquire FdwXactLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Insert MyProc into the tail of FdwXactActiveQueue.
+ */
+static void
+FdwXactQueueInsert(void)
+{
+	SHMQueueInsertBefore(&(FdwXactRslvCtl->FdwXactActiveQueue),
+						 &(MyProc->fdwXactLinks));
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * Resolve one distributed transaction. The target distributed transaction
+ * is fetched from either the active queue or the retry queue and its participants
+ * are fetched from either the global array.
+ *
+ * Release the waiter and return true if we resolved the all of the foreign
+ * transaction participants. On failure, we move the FdwXactLinks entry to the
+ * retry queue from the active queue, and raise an error and exit.
+ */
+bool
+FdwXactResolveDistributedTransaction(Oid dbid, bool is_active)
+{
+	FdwXactResolveState *state;
+	ListCell			*lc;
+	ListCell			*next;
+	PGPROC				*waiter = NULL;
+	List				*participants;
+	SHM_QUEUE			*target_queue;
+
+	if (is_active)
+		target_queue = &(FdwXactRslvCtl->FdwXactActiveQueue);
+	else
+		target_queue = &(FdwXactRslvCtl->FdwXactRetryQueue);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/* Fetch a waiter from beginning of the queue */
+	while ((waiter = (PGPROC *) SHMQueueNext(target_queue, target_queue,
+											 offsetof(PGPROC, fdwXactLinks))) != NULL)
+	{
+		/* Found a waiter */
+		if (waiter->databaseId == dbid)
+			break;
+	}
+
+	/* If no waiter, there is no job */
+	if (!waiter)
+	{
+		LWLockRelease(FdwXactLock);
+		return false;
+	}
+
+	Assert(TransactionIdIsValid(waiter->fdwXactWaitXid));
+
+	state = create_fdw_xact_resovle_state();
+	state->wait_xid = waiter->fdwXactWaitXid;
+	participants = get_fdw_xacts(dbid, waiter->fdwXactWaitXid,
+								 InvalidOid, InvalidOid, false);
+	LWLockRelease(FdwXactLock);
+
+	/* Resolve all foreign transactions one by one */
+	for (lc = list_head(participants); lc != NULL; lc = next)
+	{
+		FdwXact fdwxact = (FdwXact) lfirst(lc);
+
+		CHECK_FOR_INTERRUPTS();
+
+		next = lnext(lc);
+
+		state->serverid = fdwxact->serverid;
+		state->userid = fdwxact->userid;
+		state->umid = fdwxact->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		PG_TRY();
+		{
+			FdwXactResolveForeignTransaction(state, fdwxact, ERROR);
+		}
+		PG_CATCH();
+		{
+			/* Re-insert the waiter to the retry queue */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDW_XACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				SHMQueueInsertBefore(&(FdwXactRslvCtl->FdwXactRetryQueue),
+									 &(waiter->fdwXactLinks));
+				waiter->fdwXactState = FDW_XACT_WAITING_RETRY;
+			}
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved a foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+	}
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter
+	 * could already be detached if user cancelled to wait before
+	 * resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId	wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDW_XACT_WAIT_COMPLETE;
+
+		/* Wake up the waiter only when we have set state and removed from queue */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc xid %u", wait_xid);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	return true;
+}
+
+/*
+ * Resolve all dangling foreign transactions on the given database. Get
+ * all dangling foreign transactions from shmem global array and resolve
+ * them one by one.
+ */
+void
+FdwXactResolveAllDanglingTransactions(Oid dbid)
+{
+	List		*dangling_fdwxacts = NIL;
+	ListCell	*cell;
+	bool		n_resolved = 0;
+	int			i;
+
+	Assert(OidIsValid(dbid));
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/*
+	 * Walk over the global array to make the list of dangling transactions
+	 * of which corresponding local transaction is on the given database.
+	 */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fxact = FdwXactCtl->fdw_xacts[i];
+
+		/*
+		 * Append the fdwxact entry on the given database to the list if
+		 * it's handled by nobody and the corresponding local transaction
+		 * is not part of the prepared transaction.
+		 */
+		if (fxact->dbid == dbid &&
+			fxact->held_by == InvalidBackendId &&
+			!TwoPhaseExists(fxact->local_xid))
+			dangling_fdwxacts = lappend(dangling_fdwxacts, fxact);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/* Return if there is no foreign transaction we need to resolve */
+	if (dangling_fdwxacts == NIL)
+		return;
+
+	foreach(cell, dangling_fdwxacts)
+	{
+		FdwXact fdwxact = (FdwXact) lfirst(cell);
+		FdwXactResolveState *state;
+
+		state = create_fdw_xact_resovle_state();
+		state->wait_xid = fdwxact->local_xid;
+		state->serverid = fdwxact->serverid;
+		state->userid = fdwxact->userid;
+		state->umid = fdwxact->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		FdwXactResolveForeignTransaction(state, fdwxact, ERROR);
+
+		n_resolved++;
+	}
+
+	list_free(dangling_fdwxacts);
+
+	elog(DEBUG2, "resolved %d dangling foreign xacts", n_resolved);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ *
+ * In commit case, we have already prepared transactions on the foreign
+ * servers during pre-commit. And that prepared transactions will be
+ * resolved by the resolver process. So we don't do anything about the
+ * foreign transaction.
+ *
+ * In abort case, user requested rollback or we changed over rollback
+ * due to error during commit. To close current foreign transaction anyway
+ * we call rollback API to every foreign transaction. If we raised an error
+ * during preparing and came to here, it's possible that some entries of
+ * FdwXactParticipants already registered its FdwXact entry. If there is
+ * we leave them as dangling transaction and ask the resolver process to
+ * process them.
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		int left_fdwxacts = 0;
+		FdwXactResolveState *state = create_fdw_xact_resovle_state();
+
+		foreach (lcell, FdwXactParticipantsForAC)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * Count FdwXact entries that we registered to shared memory array
+			 * in this transaction.
+			 */
+			if (fdw_part->fdw_xact)
+			{
+				/*
+				 * The status of foreign transaction must be either preparing
+				 * or prepared. In any case, since we have registered FdwXact
+				 * entry we leave them to the resolver process. For the preparing
+				 * state, since the foreign transaction might not close yet we
+				 * fall through and call rollback API. For the prepared state,
+				 * since the foreign transaction has closed we don't need to do
+				 * anything.
+				 */
+				Assert(fdw_part->fdw_xact->status == FDW_XACT_PREPARING ||
+					   fdw_part->fdw_xact->status == FDW_XACT_PREPARED);
+
+				left_fdwxacts++;
+				if (fdw_part->fdw_xact->status == FDW_XACT_PREPARED)
+					continue;
+			}
+
+			state->serverid = fdw_part->server->serverid;
+			state->userid = fdw_part->usermapping->userid;
+			state->umid = fdw_part->usermapping->umid;
+
+			/*
+			 * Rollback all current foreign transaction. Since we're rollbacking
+			 * the transaction it's too late even if we raise an error here.
+			 * So we log it as warning.
+			 */
+			if (!fdw_part->rollback_foreign_xact(state))
+				ereport(WARNING,
+						(errmsg("could not abort transaction on server \"%s\"",
+								fdw_part->server->servername)));
+		}
+
+		/* If we left some FdwXact entries, ask the resolver process */
+		if (left_fdwxacts > 0)
+		{
+			ereport(WARNING,
+					(errmsg("might have left %u foreign transactions in in-doubt status",
+							left_fdwxacts)));
+			fdwxact_maybe_launch_resolver(true);
+		}
+	}
+
+	ForgetAllFdwXactParticipants();
+}
+
+/*
+ * AtPrepare_FdwXacts
+ *
+ * If there are foreign servers involved in the transaction, this function
+ * prepares transactions on those servers.
+ *
+ * Note that it can happen that the transaction aborts after we prepared part
+ * of participants. In this case since we can change to abort we cannot forget
+ * FdwXactParticipantsForAC here. These are processed by the resolver process
+ * during aborting, or at EOXact_FdwXacts.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipantsForAC == NIL)
+		return;
+
+	/*
+	 * We cannot prepare distributed transaction if any foreign server of
+	 * participants in the transaction isn't capable of two-phase commit.
+	 */
+	if ((MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_T_R_INTEGRITY_CONSTRAINT_VIOLATION),
+				 errmsg("can not prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * FdwXactResolveForeignTransaction
+ *
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * handler routine. The foreign transaction can be a dangling transaction
+ * that is not interested by nobody. If the fate of foreign transaction is
+ * not determined yet, it'sdetermined according to the status of corresponding
+ * local transaction.
+ *
+ * If the resolution is successful, remove the foreign transaction entry from
+ * the shared memory and also remove the corresponding on-disk file.
+ */
+static bool
+FdwXactResolveForeignTransaction(FdwXactResolveState *state, FdwXact fdwxact,
+								 int elevel)
+{
+	ForeignServer		*server;
+	ForeignDataWrapper	*fdw;
+	FdwRoutine			*fdw_routine;
+	bool		is_commit;
+	bool		ret;
+
+	Assert(fdwxact);
+
+	/*
+	 * Determine whether we commit or abort this foreign transaction.
+	 */
+	if (fdwxact->status == FDW_XACT_COMMITTING_PREPARED)
+		is_commit = true;
+	else if (fdwxact->status == FDW_XACT_ABORTING_PREPARED)
+		is_commit = false;
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	else if (TransactionIdDidCommit(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_COMMITTING_PREPARED;
+		is_commit = true;
+	}
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_ABORTING_PREPARED;
+		is_commit = false;
+	}
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+		is_commit = false;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction
+	 * state is neither committing or aborting. This should not
+	 * happen because we cannot determine to do commit or abort for
+	 * foreign transaction associated with the in-progress local
+	 * transaction.
+	 */
+	else
+		ereport(ERROR,
+				(errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid)));
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	/* Resolve the foreign transaction */
+	Assert(fdw_routine->ResolveForeignTransaction);
+
+	ret = fdw_routine->ResolveForeignTransaction(state, is_commit);
+
+	if (!ret)
+	{
+		ereport(elevel,
+				(errmsg("could not %s a prepared foreign transaction on server \"%s\"",
+						is_commit ? "commit" : "rollback", server->servername),
+				 errdetail("local transaction id is %u, connected by user id %u",
+						   fdwxact->local_xid, fdwxact->userid)));
+	}
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdw_xact(fdwxact);
+	LWLockRelease(FdwXactLock);
+
+	return ret;
+}
+
+static FdwXactResolveState *
+create_fdw_xact_resovle_state(void)
+{
+	FdwXactResolveState *state;
+
+	state = palloc(sizeof(FdwXactResolveState));
+	state->wait_xid = InvalidTransactionId;
+	state->serverid = InvalidOid;
+	state->userid = InvalidOid;
+	state->umid = InvalidOid;
+	state->fdwxact_id = NULL;
+	state->fdw_state = NULL;
+
+	return state;
+}
+
+/*
+ * Return one FdwXact entry that matches to given arguments, otherwise
+ * return NULL. Since this function search FdwXact entry by unique key
+ * all arguments should be valid.
+ */
+static FdwXact
+get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				 bool need_lock)
+{
+	List	*fdw_xact_list;
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, need_lock);
+
+	/* Could not find entry */
+	if (fdw_xact_list == NIL)
+		return NULL;
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdw_xact_list) == 1);
+
+	return (FdwXact) linitial(fdw_xact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdw_xact_exists(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdw_xact_list;
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, true);
+
+	return fdw_xact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_prepared_fdw_xacts.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdw_xacts(int *length)
+{
+	List		*all_fdw_xacts;
+	ListCell	*lc;
+	FdwXact		fdw_xacts;
+	int			num_fdw_xacts = 0;
+
+	Assert(length != NULL);
+
+	/* Get all entries */
+	all_fdw_xacts = get_fdw_xacts(InvalidOid, InvalidTransactionId,
+								  InvalidOid, InvalidOid, true);
+
+	if (all_fdw_xacts == NIL)
+	{
+		*length = 0;
+		return NULL;
+	}
+
+	fdw_xacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdw_xacts));
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdw_xacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdw_xacts + num_fdw_xacts, fx,
+			   sizeof(FdwXactData));
+		num_fdw_xacts++;
+	}
+
+	*length = num_fdw_xacts;
+	list_free(all_fdw_xacts);
+
+	return fdw_xacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return
+ * NIL.
+ */
+static List*
+get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			  bool need_lock)
+{
+	int i;
+	List	*fdw_xact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact	fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool	matches = true;
+
+		/* xid */
+		if (xid != InvalidTransactionId && xid != fdw_xact->local_xid)
+			matches = false;
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdw_xact->dbid != dbid)
+			matches = false;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdw_xact->serverid)
+			matches = false;
+
+		/* userid */
+		if (OidIsValid(userid) && fdw_xact->userid != userid)
+			matches = false;
+
+		/* Append it if matched */
+		if (matches)
+			fdw_xact_list = lappend(fdw_xact_list, fdw_xact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdw_xact_list;
+}
+
+/*
+ * fdw_xact_redo
+ * Apply the redo log for a foreign transaction.
+ */
+void
+fdw_xact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDW_XACT_REMOVE)
+	{
+		xl_fdw_xact_remove *record = (xl_fdw_xact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. Returned string
+ * value is used to identify foreign transaction. The identifier should not
+ * be same as any other concurrent prepared transaction identifier.
+ *
+ * To make the foreign transactionid, we should ideally use something like
+ * UUID, which gives unique ids with high probability, but that may be expensive
+ * here and UUID extension which provides the function to generate UUID is
+ * not part of the core code.
+ */
+static char *
+generate_fdw_xact_identifier(Oid serverid, Oid userid)
+{
+	char*	fdw_xact_id;
+
+	fdw_xact_id = (char *)palloc(FDW_XACT_ID_MAX_LEN * sizeof(char));
+
+	snprintf(fdw_xact_id, FDW_XACT_ID_MAX_LEN, "%s_%ld_%d_%d",
+			 "fx", Abs(random()), serverid, userid);
+	fdw_xact_id[strlen(fdw_xact_id)] = '\0';
+
+	return fdw_xact_id;
+}
+
+/*
+ * CheckPointFdwXact
+ *
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * In order to avoid disk I/O while holding a light weight lock, the function
+ * first collects the files which need to be synced under FdwXactLock and then
+ * syncs them after releasing the lock. This approach creates a race condition:
+ * after releasing the lock, and before syncing a file, the corresponding
+ * foreign transaction entry and hence the file might get removed. The function
+ * checks whether that's true and ignores the error if so.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdw_xacts = 0;
+
+	/* Quick get-away, before taking lock */
+	if (max_prepared_foreign_xacts <= 0)
+		return;
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/* Another quick, before we allocate memory */
+	if (FdwXactCtl->numFdwXacts <= 0)
+	{
+		LWLockRelease(FdwXactLock);
+		return;
+	}
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		FdwXact		fxact = FdwXactCtl->fdw_xacts[cnt];
+
+		if ((fxact->valid || fxact->inredo) &&
+			!fxact->ondisk &&
+			fxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fxact->dbid, fxact->local_xid,
+								fxact->serverid, fxact->userid,
+								buf, len);
+			fxact->ondisk = true;
+			fxact->insert_start_lsn = InvalidXLogRecPtr;
+			fxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdw_xacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDW_XACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdw_xacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdw_xacts,
+							 serialized_fdw_xacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDW_XACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDW_XACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * ProcessFdwXactBuffer
+ *
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid = ShmemVariableCache->nextXid;
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(insert_start_lsn != InvalidXLogRecPtr);
+
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid, true);
+		if (buf == NULL)
+		{
+			ereport(WARNING,
+					(errmsg("removing corrupt fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+			return NULL;
+		}
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return thecontents in
+ * a structure allocated in-memory. Otherwise return NULL. The structure can
+ * be later freed by the caller.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				bool give_warnings)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+	{
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not stat FDW transaction state file \"%s\": %m",
+							path)));
+		return NULL;
+	}
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdw_xact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+	{
+		CloseTransientFile(fd);
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+		return NULL;
+	}
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+	{
+		CloseTransientFile(fd);
+		return NULL;
+	}
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_READ);
+	if (read(fd, buf, stat.st_size) != stat.st_size)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not read FDW transaction state file \"%s\": %m",
+					  path)));
+		return NULL;
+	}
+
+	pgstat_report_wait_end();
+	CloseTransientFile(fd);
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+	{
+		pfree(buf);
+		return NULL;
+	}
+
+	/* Check if the contents is an expected data */
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fxact_file_data->dbid  != dbid ||
+		fxact_file_data->serverid != serverid ||
+		fxact_file_data->userid != userid ||
+		fxact_file_data->local_xid != xid)
+	{
+		ereport(WARNING,
+			(errmsg("invalid foreign transaction state file \"%s\"",
+					path)));
+		CloseTransientFile(fd);
+		pfree(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+/*
+ * PrescanFdwXacts
+ *
+ * Scan the all foreign transactions directory for oldest active transaction.
+ * This is run during database startup, after we completed reading WAL.
+ * ShmemVariableCache->nextXid has been set to one more than the highest XID
+ * for which evidence exists in WAL.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	TransactionId nextXid = ShmemVariableCache->nextXid;
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+		 strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			TransactionId local_xid;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/*
+			 * Remove a foreign prepared transaction file corresponding to an
+			 * XID, which is too new.
+			 */
+			if (TransactionIdFollowsOrEquals(local_xid, nextXid))
+			{
+				ereport(WARNING,
+						(errmsg("removing future foreign prepared transaction file \"%s\"",
+								clde->d_name)));
+				RemoveFdwXactFile(dbid, local_xid, serverid, userid, true);
+				continue;
+			}
+
+			if (TransactionIdPrecedesOrEquals(local_xid, oldestActiveXid))
+				oldestActiveXid = local_xid;
+		}
+	}
+
+	FreeDir(cldir);
+	return oldestActiveXid;
+}
+
+/*
+ * restoreFdwXactData
+ *
+ * Scan pg_fdw_xact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid, bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * FdwXactRedoAdd
+ *
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fxact = insert_fdw_xact(fxact_data->dbid, fxact_data->local_xid,
+							fxact_data->serverid, fxact_data->userid,
+							fxact_data->umid, fxact_data->fdw_xact_id);
+
+	/*
+	 * Set status as preparing, since we do not know the xact status
+	 * right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->insert_start_lsn = start_lsn;
+	fxact->insert_end_lsn = end_lsn;
+	fxact->inredo = true;	/* added in redo */
+	fxact->valid = false;
+	fxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * FdwXactRedoRemove
+ *
+ * Remove the corresponding fdw_xact entry from FdwXactCtl.
+ * Also remove fdw_xact file if a foreign transaction was saved
+ * via an earlier checkpoint.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdw_xact(dbid, xid, serverid, userid,
+							   false);
+
+	if (fdwxact == NULL)
+		return;
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdw_xact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(bool *newval, void **extra, GucSource source)
+{
+	/* Parameter check */
+	if (*newval &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdw_xacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_prepared_fdw_xacts(PG_FUNCTION_ARGS)
+{
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdw_xacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdw_xacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(6, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdw_xacts = get_all_fdw_xacts(&num_fdw_xacts);
+		status->num_xacts = num_fdw_xacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdw_xact = &status->fdw_xacts[status->cur_xact++];
+		Datum		values[6];
+		bool		nulls[6];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdw_xact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdw_xact->dbid);
+		values[1] = TransactionIdGetDatum(fdw_xact->local_xid);
+		values[2] = ObjectIdGetDatum(fdw_xact->serverid);
+		values[3] = ObjectIdGetDatum(fdw_xact->userid);
+		switch (fdw_xact->status)
+		{
+			case FDW_XACT_PREPARING:
+				xact_status = "prepared";
+				break;
+			case FDW_XACT_COMMITTING_PREPARED:
+				xact_status = "committing";
+				break;
+			case FDW_XACT_ABORTING_PREPARED:
+				xact_status = "aborting";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		/* should this be really interpreted by FDW */
+		values[5] = PointerGetDatum(cstring_to_text_with_len(fdw_xact->fdw_xact_id,
+															 strlen(fdw_xact->fdw_xact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXactResolveState *state;
+	UserMapping		*usermapping;
+	FdwXact			fdwxact;
+	bool			ret;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, true);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	usermapping = GetUserMapping(userid, serverid);
+
+	state = create_fdw_xact_resovle_state();
+	state->wait_xid = xid;
+	state->serverid = serverid;
+	state->userid = userid;
+	state->umid = usermapping->umid;
+
+	ret = FdwXactResolveForeignTransaction(state, fdwxact, LOG);
+
+	PG_RETURN_BOOL(ret);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, false);
+	if (fdwxact == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("could not find foreign transaction entry"))));
+
+	remove_fdw_xact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_VOID();
+}
diff --git a/src/backend/access/fdwxact/fdwxact_launcher.c b/src/backend/access/fdwxact/fdwxact_launcher.c
new file mode 100644
index 0000000..39f351b
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact_launcher.c
@@ -0,0 +1,641 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * There is a shared memory area where the information of resolver process
+ * is stored. Requesting of starting new resolver process by backend process
+ * is done via that shared memory area. Note that the launcher is assuming
+ * that there is no more than one starting request for a database.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/fdwxact_launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launcher_sigusr2(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid, int slot);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+Datum pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS);
+
+/*
+ * Wake up the launcher process to retry launch. This is used by
+ * the resolver process is being stopped.
+ */
+void
+FdwXactLauncherWakeupToRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request resolution. This is
+ * used by the backend process.
+ */
+void
+FdwXactLauncherWakeupToRequest(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->FdwXactActiveQueue));
+		SHMQueueInit(&(FdwXactRslvCtl->FdwXactRetryQueue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR1: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a foreign_xact_resolution_retry_interval
+		 * but always try to start by the backend request.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool launched;
+
+			ResetLatch(MyLatch);
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher launch",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver worker
+ * if not running yet. A foreign transaction resolver worker is responsible
+ * for resolution of foreign transaction that are registered on a database.
+ * So if a resolver worker already is launched, we don't need to launch new
+ * one.
+ */
+void
+fdwxact_maybe_launch_resolver(bool ignore_error)
+{
+	FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->pid != InvalidPid &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/*
+	 * If we found the resolver for my database, we don't need to launch new
+	 * one but wake running worker up.
+	 */
+	if (found)
+	{
+		SetLatch(resolver->latch);
+
+		elog(DEBUG1, "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		return;
+	}
+
+	/* Looking for unused resolver slot */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	/*
+	 * However if there are no more free worker slots, inform user about it before
+	 * exiting.
+	 */
+	if (!found)
+	{
+		LWLockRelease(FdwXactResolverLock);
+
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+		return;
+	}
+
+	Assert(resolver->pid == InvalidPid);
+
+	/* Found a new resolver process */
+	resolver->dbid = MyDatabaseId;
+	resolver->in_use = true;
+
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Wake up launcher */
+	FdwXactLauncherWakeupToRequest();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid' at 'slot' if given. If slot is negative value we find an unused slot.
+ * Note that caller must hold FdwXactResolverLock in exclusive mode.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid, int slot)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int launch_slot = slot;
+
+	/* If slot number is invalid, we find an unused slot */
+	if (launch_slot < 0)
+	{
+		int i;
+
+		for (i = 0; i < max_foreign_xact_resolvers; i++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+			if (resolver->in_use && resolver->dbid == dbid)
+				return;
+
+			if (!resolver->in_use)
+			{
+				launch_slot = i;
+				break;
+			}
+		}
+	}
+
+	/* No unused found */
+	if (launch_slot < 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[launch_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_main_arg = Int32GetDatum(launch_slot);
+	bgw.bgw_notify_pid = (Datum) 0;
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch all foreign transaction resolvers that are required by backend process
+ * but not running. Return true if we launch any resolver.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	int i, j;
+	int num_launches = 0;
+	int num_unused_slots = 0;
+	int num_dbs = 0;
+	bool launched = false;
+	Oid	*dbs_to_launch;
+	Oid	*dbs_having_worker = palloc0(sizeof(Oid) * max_foreign_xact_resolvers);
+
+	/*
+	 * Launch resolver workers on the databases that are requested
+	 * by backend processes while looking unused slots.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* Remember unused worker slots */
+		if (!resolver->in_use)
+		{
+			num_unused_slots++;
+			continue;
+		}
+
+		/* Remember databases that are having a resolve worker, fall through */
+		if (OidIsValid(resolver->dbid))
+			dbs_having_worker[num_dbs++] = resolver->dbid;
+
+		/* Launch the backend-requested worker */
+		if (resolver->in_use &&
+			OidIsValid(resolver->dbid) &&
+			resolver->pid == InvalidPid)
+		{
+			fdwxact_launch_resolver(resolver->dbid, i);
+			launched = true;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* quick exit if no unused slot */
+	if (num_unused_slots == 0)
+		return launched;
+
+	/*
+	 * Launch the stopped resolver on the database that has unresolved
+	 * foreign transaction but doesn't have any resolver. Scanning
+	 * all FdwXact entries could take time but it's harmless for the
+	 * relaunch case.
+	 */
+	dbs_to_launch = (Oid *) palloc(sizeof(Oid) * num_unused_slots);
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool found = false;
+
+		/* unused slot is full */
+		if (num_launches > num_unused_slots)
+			break;
+
+		for (j = 0; j < num_dbs; j++)
+		{
+			if (dbs_having_worker[j] == fdw_xact->dbid)
+			{
+				found = true;
+				break;
+			}
+		}
+
+		/* Register the database if any resolvers aren't working on that */
+		if (!found)
+			dbs_to_launch[num_launches++] = fdw_xact->dbid;
+	}
+
+	/* Launch resolver process for a database at any worker slot */
+	for (i = 0; i < num_launches; i++)
+	{
+		fdwxact_launch_resolver(dbs_to_launch[i], -1);
+		launched = true;
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+
+/*
+ * Returns activity of foreign transaction resolvers, including pids, the number
+ * of tasks and the last resolution time.
+ */
+Datum
+pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/fdwxact_resolver.c b/src/backend/access/fdwxact/fdwxact_resolver.c
new file mode 100644
index 0000000..0b754da
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact_resolver.c
@@ -0,0 +1,331 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for every databases.
+ *
+ * A resolver process continues to resolve foreign transactions on a database
+ * It resolves two types of foreign transactions: on-line foreign transaction
+ * and dangling foreign transaction. The on-line foreign transaction is a
+ * foreign transaction that a concurrent backend process is waiting for
+ * resolution. The dangling transaction is a foreign transaction that corresponding
+ * distributed transaction ended up in in-doubt state. A resolver process
+ * doesn' exit as long as there is at least one unresolved foreign transaction
+ * on the database even if the timeout has come.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/fdwxact_resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+
+//static MemoryContext ResolveContext = NULL;
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FdwXactRslvLoop(void);
+static long FdwXactRslvComputeSleepTime(TimestampTz now);
+static void FdwXactRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+	FdwXactLauncherWakeupToRetry();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	TIMESTAMP_NOBEGIN(MyFdwXactResolver->last_resolved_time);
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FdwXactRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FdwXactRslvLoop(void)
+{
+	TimestampTz last_retry_time = 0;
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time;
+		bool		resolved;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		/* Resolve one distributed transaction */
+		StartTransactionCommand();
+		resolved = FdwXactResolveDistributedTransaction(MyDatabaseId, true);
+		CommitTransactionCommand();
+
+		now = GetCurrentTimestamp();
+
+		/* Update my state */
+		if (resolved)
+			MyFdwXactResolver->last_resolved_time = now;
+
+		if (TimestampDifferenceExceeds(last_retry_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			StartTransactionCommand();
+			resolved = FdwXactResolveDistributedTransaction(MyDatabaseId, false);
+			CommitTransactionCommand();
+
+			last_retry_time = GetCurrentTimestamp();
+
+			/* Update my state */
+			if (resolved)
+				MyFdwXactResolver->last_resolved_time = last_retry_time;
+		}
+
+		/* Check for fdwxact resolver timeout */
+		FdwXactRslvCheckTimeout(now);
+
+		/*
+		 * If we have resolved any distributed transaction we go the next
+		 * without both resolving dangling transaction and sleeping because
+		 * there might be other on-line transactions waiting to be resolved.
+		 */
+		if (!resolved)
+		{
+			/* Resolve dangling transactions as mush as possible */
+			StartTransactionCommand();
+			FdwXactResolveAllDanglingTransactions(MyDatabaseId);
+			CommitTransactionCommand();
+
+			sleep_time = FdwXactRslvComputeSleepTime(now);
+
+			MemoryContextResetAndDeleteChildren(resolver_ctx);
+			MemoryContextSwitchTo(TopMemoryContext);
+
+			rc = WaitLatch(MyLatch,
+						   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+						   sleep_time,
+						   WAIT_EVENT_FDW_XACT_RESOLVER_MAIN);
+
+			if (rc & WL_POSTMASTER_DEATH)
+				proc_exit(1);
+		}
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FdwXactRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	/*
+	 * Reached to the timeout. We exit if there is no more both pending on-line
+	 * transactions and dangling transactions.
+	 */
+	if (!fdw_xact_exists(InvalidTransactionId, MyDatabaseId, InvalidOid,
+						 InvalidOid))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyFdwXactResolver->dbid))));
+		CommitTransactionCommand();
+
+		fdwxact_resolver_detach();
+		proc_exit(0);
+	}
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. Return the sleep time
+ * in milliseconds, -1 means that we reached to the timeout and should exits
+ */
+static long
+FdwXactRslvComputeSleepTime(TimestampTz now)
+{
+	static TimestampTz	wakeuptime = 0;
+	long	sleeptime;
+	long	sec_to_timeout;
+	int		microsec_to_timeout;
+
+	if (now >= wakeuptime)
+		wakeuptime = TimestampTzPlusMilliseconds(now,
+												 foreign_xact_resolution_retry_interval);
+
+	/* Compute relative time until wakeup. */
+	TimestampDifference(now, wakeuptime,
+						&sec_to_timeout, &microsec_to_timeout);
+
+	sleeptime = sec_to_timeout * 1000 + microsec_to_timeout / 1000;
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index 5514db1..742e825 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -8,9 +8,9 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
-	   gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
-	   mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
-	   smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdwxactdesc.o \
+	genericdesc.o  gindesc.o gistdesc.o hashdesc.o heapdesc.o \
+	logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \
+	seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000..7061bba
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,65 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdw_xactdesc.c
+ *		PostgreSQL distributed transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdw_xactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdw_xact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		FdwXactOnDiskData *fdw_insert_xlog = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_insert_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_insert_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_insert_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_insert_xlog->local_xid);
+		/* TODO: This should be really interpreted by each FDW */
+
+		/*
+		 * TODO: we also need to assess whether we want to add this
+		 * information
+		 */
+		appendStringInfo(buf, " foreign transaction info: %s",
+						 fdw_insert_xlog->fdw_xact_id);
+	}
+	else
+	{
+		xl_fdw_xact_remove *fdw_remove_xlog = (xl_fdw_xact_remove *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_remove_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_remove_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_remove_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_remove_xlog->xid);
+	}
+
+}
+
+const char *
+fdw_xact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDW_XACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDW_XACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 00741c7..4a9ab3d 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -112,14 +112,16 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_prepared_xacts=%d max_locks_per_xact=%d "
 						 "wal_level=%s wal_log_hints=%s "
-						 "track_commit_timestamp=%s",
+						 "track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_prepared_xacts,
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47..f15c83a 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -12,9 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
-	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
-	xact.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o generic_xlog.o multixact.o \
+	parallel.o rmgr.o slru.o subtrans.o timeline.o transam.o twophase.o \
+	twophase_rmgr.o varsup.o xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9368b56..8b360b1 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -9,6 +9,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
 #include "access/generic_xlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 3942734..bc4e109 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -844,6 +845,35 @@ TwoPhaseGetGXact(TransactionId xid)
 }
 
 /*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
+/*
  * TwoPhaseGetDummyProc
  *		Get the dummy backend ID for prepared transaction specified by XID
  *
@@ -2316,6 +2346,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2375,6 +2411,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 8c1621d..b2b3c89 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1131,6 +1132,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_twophase_for_ac;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1139,6 +1141,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_twophase_for_ac = ForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1177,12 +1180,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_twophase_for_ac)
 			goto cleanup;
 	}
 	else
@@ -1340,6 +1344,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_twophase_for_ac && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -1990,6 +2002,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2146,6 +2161,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2233,6 +2249,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2422,6 +2440,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2627,6 +2646,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7375a78..9de3bcc 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
 #include "access/subtrans.h"
@@ -5267,6 +5268,7 @@ BootStrapXLOG(void)
 	ControlFile->MaxConnections = MaxConnections;
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6354,6 +6356,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -6878,14 +6883,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdw_xact, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -7077,7 +7083,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7583,6 +7592,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7901,6 +7911,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9217,6 +9230,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9650,7 +9664,8 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9682,6 +9697,7 @@ XLogReportParameters(void)
 		ControlFile->MaxConnections = MaxConnections;
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9887,6 +9903,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -10085,6 +10102,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->MaxConnections = xlrec.MaxConnections;
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index a03b005..47e9317 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -291,6 +291,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_prepared_fdw_xacts AS
+       SELECT * FROM pg_prepared_fdw_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
 	l.objoid, l.classoid, l.objsubid,
@@ -773,6 +776,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_fdwxact_resolvers AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_fdwxact_resolver() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index e5dd995..dac1e3a 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
@@ -1093,6 +1094,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1407,6 +1420,16 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
 	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srv->serverid,
+						useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
+	/*
 	 * Do the deletion
 	 */
 	object.classId = UserMappingRelationId;
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index 0bcb237..e063922 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "catalog/partition.h"
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_type.h"
@@ -749,7 +750,10 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+		FdwXactMarkForeignTransactionModified(partRelInfo, 0);
+	}
 
 	MemoryContextSwitchTo(oldContext);
 
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 2ec7fcb..a16e1e4 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,7 +226,13 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+
+		/* Mark this transaction modified data on the foreign server */
+		FdwXactMarkForeignTransactionModified(estate->es_result_relation_info,
+										 eflags);
+	}
 	else
 		fdwroutine->BeginForeignScan(scanstate, eflags);
 
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 528f587..4ef20b7 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "commands/trigger.h"
@@ -44,6 +45,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "storage/bufmgr.h"
@@ -2321,6 +2323,9 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 															 fdw_private,
 															 i,
 															 eflags);
+
+			/* Mark this transaction modified data on the foreign server */
+			FdwXactMarkForeignTransactionModified(resultRelInfo, eflags);
 		}
 
 		resultRelInfo++;
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index a0bcc04..b2097ad 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -155,6 +155,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMapping - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index d2b695e..b722b9a 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -15,6 +15,8 @@
 #include <unistd.h>
 
 #include "libpq/pqsignal.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 8de603d..3f59fdd 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3484,6 +3484,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_LAUNCHER_MAIN:
 			event_name = "LogicalLauncherMain";
 			break;
@@ -3675,6 +3681,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -3890,6 +3899,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDW_XACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 688f462..883ad85 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -896,6 +898,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -971,12 +977,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index afb4972..960fd6a 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -154,6 +154,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDW_XACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 0c86a58..c5610ee 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -150,6 +152,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
 		size = add_size(size, BackendRandomShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -270,6 +274,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 	SyncScanShmemInit();
 	AsyncShmemInit();
 	BackendRandomShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 908f62d..cc578b2 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -90,6 +90,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -245,6 +247,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1323,6 +1326,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	volatile TransactionId replication_slot_xmin = InvalidTransactionId;
 	volatile TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	volatile TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1384,6 +1388,7 @@ GetOldestXmin(Relation rel, int flags)
 	/* fetch into volatile var while ProcArrayLock is held */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1434,6 +1439,15 @@ GetOldestXmin(Relation rel, int flags)
 		result = replication_slot_xmin;
 
 	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDW_XACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
+	/*
 	 * After locks have been released and defer_cleanup_age has been applied,
 	 * check whether we need to back up further to make logical decoding
 	 * possible. We need to do so if we're computing the global limit (rel =
@@ -3016,6 +3030,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index e6025ec..a42d06e 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -50,3 +50,5 @@ OldSnapshotTimeMapLock				42
 BackendRandomLock					43
 LogicalRepWorkerLock				44
 CLogTruncationLock					45
+FdwXactLock					46
+FdwXactResolverLock			47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 6f9aaa5..8e55dad 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -398,6 +399,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* initialize fields for fdw xact */
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -799,6 +804,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 6e13d14..1302d16 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -2971,6 +2973,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 2317e8b..0c371fe 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/transam.h"
@@ -659,6 +660,10 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -1831,6 +1836,16 @@ static struct config_bool ConfigureNamesBool[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+			gettext_noop("Sets the usage of two-phase commit protocol for distributed transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		false,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
 	/* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL
@@ -2235,6 +2250,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 4e61bc6..88cdc85 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -121,6 +121,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -287,6 +289,20 @@
 
 
 #------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
+#------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index ad06e8e..ca3eb62 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index ab5cb7f..609578c 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -209,6 +209,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdw_xact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 895a51f..7df88e0 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -306,6 +306,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_worker_processes);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 6fb403a..6d867c8 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -730,6 +730,7 @@ GuessControlValues(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -957,6 +958,7 @@ RewriteControlFile(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* Contents are protected with a CRC */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca..b616cea 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000..9563a1e
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,150 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL distributed transaction manager
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDW_XACT_H
+#define FDW_XACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+#define	FDW_XACT_NOT_WAITING		0
+#define	FDW_XACT_WAITING			1
+#define	FDW_XACT_WAITING_RETRY		2
+#define	FDW_XACT_WAIT_COMPLETE		3
+
+#define FdwXactEnabled() (max_prepared_foreign_xacts > 0)
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDW_XACT_ID_MAX_LEN 200
+
+/* Enum to track the status of prepared foreign transaction */
+typedef enum
+{
+	FDW_XACT_INITIAL,
+	FDW_XACT_PREPARING,					/* foreign transaction is being prepared */
+	FDW_XACT_PREPARED,					/* foreign transaction is prepared */
+	FDW_XACT_COMMITTING_PREPARED,		/* foreign prepared transaction is to
+										 * be committed */
+	FDW_XACT_ABORTING_PREPARED, /* foreign prepared transaction is to be
+								 * aborted */
+} FdwXactStatus;
+
+/* Shared memory entry for a prepared or being prepared foreign transaction */
+typedef struct FdwXactData *FdwXact;
+
+typedef struct FdwXactData
+{
+	FdwXact		fxact_free_next;	/* Next free FdwXact entry */
+	FdwXact		fxact_next;			/* Pointer to the neext FdwXact entry accosiated
+									 * with the same transaction */
+	Oid				dbid;			/* database oid where to find foreign server
+									 * and user mapping */
+	TransactionId	local_xid;		/* XID of local transaction */
+	Oid				serverid;		/* foreign server where transaction takes place */
+	Oid				userid;			/* user who initiated the foreign transaction */
+	Oid				umid;
+	FdwXactStatus 	status;			/* The state of the foreign transaction. This
+									 * doubles as the action to be taken on this entry. */
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;		/* XLOG offset of inserting this entry start */
+	XLogRecPtr	insert_end_lsn;		/* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN];		/* prepared transaction identifier */
+} FdwXactData;
+
+/* Shared memory layout for maintaining foreign prepared transaction entries. */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		freeFdwXacts;
+
+	/* Number of valid foreign transaction entries */
+	int			numFdwXacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdw_xacts[FLEXIBLE_ARRAY_MEMBER];		/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* Struct for foreign transaction passed to API */
+typedef struct FdwXactResolveState
+{
+	Oid		serverid;
+	Oid		userid;
+	Oid		umid;
+
+	/*
+	 * The following fields are used only for COMMIT/ROLLBACK PREPARED
+	 * and PREPARE callbacks.
+	 */
+	char			*fdwxact_id;
+	void			*fdw_state;		/* foreign-data wrapper can keep state here */
+
+	/*
+	 * The following fields are used only for COMMIT/ROLLBACK PREPARED
+	 * callbacks.
+	 */
+	TransactionId	wait_xid;
+} FdwXactResolveState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern bool foreign_twophase_commit;
+
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern bool fdw_xact_exists(TransactionId xid, Oid dboid, Oid serverid,
+				Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern void PreCommit_FdwXacts(void);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern bool FdwXactResolveDistributedTransaction(Oid dbid, bool is_active);
+extern void FdwXactResolveAllDanglingTransactions(Oid dbid);
+extern bool ForeignTwophaseCommitRequired(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void FdwXactMarkForeignTransactionModified(ResultRelInfo *resultRelInfo,
+												  int flags);
+extern bool check_foreign_twophase_commit(bool *newval, void **extra,
+										  GucSource source);
+
+#endif   /* FDW_XACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000..4ea65b2
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,32 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _FDWXACT_LAUNCHER_H
+#define _FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherWakeupToRequest(void);
+extern void FdwXactLauncherWakeupToRetry(void);
+
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+
+extern bool IsFdwXactLauncher(void);
+
+extern void fdwxact_maybe_launch_resolver(bool ignore_error);
+
+
+#endif	/* _FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000..6b2a24f
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int foreign_xact_resolver_timeout;
+
+#endif		/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000..e92b5a1
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,52 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDW_XACT_INSERT	0x00
+#define XLOG_FDW_XACT_REMOVE	0x10
+
+/* Same as GIDSIZE */
+#define FDW_XACT_MAX_ID_LEN 200
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdw_xact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+} xl_fdw_xact_remove;
+
+extern void fdw_xact_redo(XLogReaderState *record);
+extern void fdw_xact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdw_xact_identify(uint8 info);
+
+#endif	/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000..36391d4
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,67 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/resovler_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _RESOLVER_INTERNAL_H
+#define _RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t	pid;	/* this resolver's PID, or 0 if not active */
+	Oid		dbid;	/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool	in_use;
+
+	/* Stats */
+	TimestampTz	last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t	mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	*latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/*
+	 * Foreign transaction resolution queues. Protected by FdwXactLock.
+	 */
+	SHM_QUEUE	FdwXactActiveQueue;
+	SHM_QUEUE	FdwXactRetryQueue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch		*launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif	/* _RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 0bbe9879..c15dff7 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDW_XACT_ID, "Foreign Transactions", fdw_xact_redo, fdw_xact_desc, fdw_xact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 0e932da..b199c88 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 				TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 2c1b2d8..63c833d 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -105,6 +105,13 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE				(1U << 3)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 30610b3..795e85a 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -227,6 +227,7 @@ typedef struct xl_parameter_change
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index 773d9e6..3d5333a 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -178,6 +178,7 @@ typedef struct ControlFileData
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index cff58ed..d39ca1e 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5032,6 +5032,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '6053', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_fdwxact_resolver', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_fdwxact_resolver' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5737,6 +5744,22 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '6050', descr => 'view foreign transactions',
+  proname => 'pg_prepared_fdw_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,text}',
+  proargmodes => '{o,o,o,o,o,o}',
+  proargnames => '{dbid,transaction,serverid,userid,status,identifier}',
+  prosrc => 'pg_prepared_fdw_xacts' },
+{ oid => '6051', descr => 'remove foreign transaction',
+  proname => 'pg_remove_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_remove_fdw_xact' },
+{ oid => '6052', descr => 'resolve foreign transaction',
+  proname => 'pg_resolve_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_resolve_fdw_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index c14eb54..c5e481a 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/relation.h"
@@ -168,6 +169,12 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef bool (*PrepareForeignTransaction_function) (FdwXactResolveState *state);
+typedef bool (*CommitForeignTransaction_function) (FdwXactResolveState *state);
+typedef bool (*RollbackForeignTransaction_function) (FdwXactResolveState *state);
+typedef bool (*ResolveForeignTransaction_function) (FdwXactResolveState *state,
+													bool is_commit);
+typedef bool (*IsTwoPhaseCommitEnabled_function) (Oid serverid);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -235,6 +242,13 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for distributed transactions */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	ResolveForeignTransaction_function ResolveForeignTransaction;
+	IsTwoPhaseCommitEnabled_function IsTwoPhaseCommitEnabled;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
@@ -247,7 +261,6 @@ typedef struct FdwRoutine
 	ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild;
 } FdwRoutine;
 
-
 /* Functions in foreign/foreign.c */
 extern FdwRoutine *GetFdwRoutine(Oid fdwhandler);
 extern Oid	GetForeignServerIdByRelId(Oid relid);
@@ -258,4 +271,7 @@ extern bool IsImportableForeignTable(const char *tablename,
 						 ImportForeignSchemaStmt *stmt);
 extern Path *GetExistingLocalJoinPath(RelOptInfo *joinrel);
 
+/* Functions in foreign/fdwxact.c */
+extern void FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, char *fdwxact_id);
+
 #endif							/* FDWAPI_H */
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 3ca12e6..d030368 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -68,10 +68,10 @@ typedef struct ForeignTable
 	List	   *options;		/* ftoptions as DefElem list */
 } ForeignTable;
 
-
 extern ForeignServer *GetForeignServer(Oid serverid);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperByName(const char *name,
 							bool missing_ok);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index d59c24a..f74d1be 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -759,6 +759,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDW_XACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -832,7 +834,8 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDW_XACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -912,6 +915,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_READ,
+	WAIT_EVENT_FDW_XACT_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index cb613c8..45880b2 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -153,6 +153,16 @@ struct PGPROC
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction
+								 * resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+
+	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
 	 * their lock.
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index 75bab29..25d6a2f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDW_XACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -124,4 +126,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 								TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 668d9ef..81560bd 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -94,6 +94,8 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 735dd37..fdd6ded 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1413,6 +1413,13 @@ pg_policies| SELECT n.nspname AS schemaname,
    FROM ((pg_policy pol
      JOIN pg_class c ON ((c.oid = pol.polrelid)))
      LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace)));
+pg_prepared_fdw_xacts| SELECT f.dbid,
+    f.transaction,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.identifier
+   FROM pg_prepared_fdw_xacts() f(dbid, transaction, serverid, userid, status, identifier);
 pg_prepared_statements| SELECT p.name,
     p.statement,
     p.prepare_time,
@@ -1821,6 +1828,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_fdwxact_resolvers| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_fdwxact_resolver() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_progress_vacuum| SELECT s.pid,
     s.datid,
     d.datname,
-- 
2.10.5

From 555fec86f082a092725fbae1c85a4e00d70f8539 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Thu, 8 Feb 2018 11:26:46 +0900
Subject: [PATCH v20 1/4] Keep track of writing on non-temporary relation.

---
 src/backend/access/heap/heapam.c | 12 ++++++++++++
 src/include/access/xact.h        |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index fb63471..c2db19b 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2629,6 +2629,10 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 		heap_freetuple(heaptup);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleGetOid(tup);
 }
 
@@ -3453,6 +3457,10 @@ l1:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleMayBeUpdated;
 }
 
@@ -4403,6 +4411,10 @@ l2:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	bms_free(hot_attrs);
 	bms_free(proj_idx_attrs);
 	bms_free(key_attrs);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 689c57c..2c1b2d8 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -98,6 +98,11 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
-- 
2.10.5

From 179365b51680f8b671720e1a56a13ebb0df0c438 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:46:01 +0900
Subject: [PATCH v20 3/4] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/connection.c              | 673 ++++++++++++++++---------
 contrib/postgres_fdw/expected/postgres_fdw.out | 387 +++++++++++++-
 contrib/postgres_fdw/option.c                  |   5 +-
 contrib/postgres_fdw/postgres_fdw.c            |  60 ++-
 contrib/postgres_fdw/postgres_fdw.h            |  11 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql      | 151 +++++-
 doc/src/sgml/postgres-fdw.sgml                 |  37 ++
 7 files changed, 1069 insertions(+), 255 deletions(-)

diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index fe4893a..3264300 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -14,9 +14,12 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
-#include "catalog/pg_user_mapping.h"
 #include "access/xact.h"
+#include "catalog/pg_user_mapping.h"
+#include "commands/defrem.h"
+#include "foreign/fdwapi.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -45,7 +48,7 @@
  */
 typedef Oid ConnCacheKey;
 
-typedef struct ConnCacheEntry
+struct ConnCacheEntry
 {
 	ConnCacheKey key;			/* hash key (must be first) */
 	PGconn	   *conn;			/* connection to foreign server, or NULL */
@@ -56,9 +59,19 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		am_participant_of_ac;	/* true if fdwxact code control the transaction */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
-} ConnCacheEntry;
+};
+
+typedef struct PgFdwXactState
+{
+	Oid		serverid;
+	Oid		userid;
+	Oid		umid;
+	ConnCacheEntry	*conn;
+} PgFdwXactState;
 
 /*
  * Connection cache (initialized on first use)
@@ -69,17 +82,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 					   SubTransactionId mySubid,
 					   SubTransactionId parentSubid,
@@ -91,24 +100,43 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 						 bool ignore_errors);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 						 PGresult **result);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
+{
+	ConnCacheEntry *entry;
+	ConnCacheKey	key;
+	bool			found;
 
-/*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
- */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
+
+	/*
+	 * Find or create cached entry for requested connection.
+	 */
+	entry = hash_search(ConnectionHash, &key, HASH_ENTER, &found);
+
+	if (!found)
+	{
+		/*
+		 * We need only clear "conn" here; remaining fields will be filled
+		 * later when "conn" is set.
+		 */
+		entry->conn = NULL;
+	}
+
+	return entry;
+}
+
+
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +156,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,24 +163,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
-	/*
-	 * Find or create cached entry for requested connection.
-	 */
-	entry = hash_search(ConnectionHash, &key, HASH_ENTER, &found);
-	if (!found)
-	{
-		/*
-		 * We need only clear "conn" here; remaining fields will be filled
-		 * later when "conn" is set.
-		 */
-		entry->conn = NULL;
-	}
+	entry = GetConnectionCacheEntry(umid);
 
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
@@ -182,6 +192,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping		*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +201,8 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->am_participant_of_ac = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +213,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,16 +229,46 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
 /*
  * Connect to remote server using specified server and user mapping properties.
+ * If the attempt to connect fails, and the caller can handle connection failure
+ * (connection_error_ok = true) return NULL, throw error otherwise.
  */
 static PGconn *
 connect_pg_server(ForeignServer *server, UserMapping *user)
@@ -265,11 +317,22 @@ connect_pg_server(ForeignServer *server, UserMapping *user)
 
 		conn = PQconnectdbParams(keywords, values, false);
 		if (!conn || PQstatus(conn) != CONNECTION_OK)
+		{
+			char	   *connmessage;
+			int			msglen;
+
+			/* libpq typically appends a newline, strip that */
+			connmessage = pstrdup(PQerrorMessage(conn));
+			msglen = strlen(connmessage);
+			if (msglen > 0 && connmessage[msglen - 1] == '\n')
+				connmessage[msglen - 1] = '\0';
+
 			ereport(ERROR,
 					(errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION),
 					 errmsg("could not connect to server \"%s\"",
 							server->servername),
 					 errdetail_internal("%s", pchomp(PQerrorMessage(conn)))));
+		}
 
 		/*
 		 * Check that non-superuser has used password to establish connection;
@@ -414,15 +477,20 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
+	ForeignServer	*server = GetForeignServer(serverid);
 
 	/* Start main transaction if we haven't yet */
 	if (entry->xact_depth <= 0)
 	{
 		const char *sql;
 
+		/* Register the new foreign server if enabled */
+		if (server_uses_twophase_commit(server))
+			entry->am_participant_of_ac = true;
+
 		elog(DEBUG3, "starting remote transaction on connection %p",
 			 entry->conn);
 
@@ -644,193 +712,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 }
 
 /*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow remote transactions that modified anything,
-					 * since it's not very reasonable to hold them open until
-					 * the prepared transaction is committed.  For the moment,
-					 * throw error unconditionally; later we might allow
-					 * read-only cases.  Note that the error will cause us to
-					 * come right back here with event == XACT_EVENT_ABORT, so
-					 * we'll clean up the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot prepare a transaction that modified remote tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
-/*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
 static void
@@ -846,10 +727,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -860,6 +737,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			return;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1193,3 +1074,327 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * The function prepares transaction on foreign server. This function
+ * is called only at the pre-commit phase of the local transaction. Since
+ * we should have the connection to the server that we are interested in
+ * we don't use serverid and userid that are necessary to get user mapping
+ * that is the key of the connection cache.
+ */
+bool
+postgresPrepareForeignTransaction(FdwXactState *state)
+{
+	PgFdwXactState *rstate;
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+	PGresult	*res;
+	StringInfo	command;
+
+	entry = GetConnectionState(state->umid, false, false);
+	//entry = hash_search(ConnectionHash, &(state->umid), HASH_FIND, NULL);
+
+	if (!entry->xact_got_connection || !entry->conn)
+		return true;
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	rstate = (PgFdwXactState *) palloc0(sizeof(PgFdwXactState));
+	rstate->serverid = state->serverid;
+	rstate->userid = state->userid;
+	rstate->umid = state->umid;
+	rstate->conn = entry;
+	state->fdw_state = (void *)rstate;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	if (result)
+		elog(DEBUG1, "prepared foreign transaction on server %u with ID %s",
+			 state->serverid, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/*
+ * The function commits the transactionon foreign server. This
+ * function is called both at the pre-commit phase of the local transaction
+ * when committing and at the end of the local transaction when aborting.
+ * Since we should the connections to the server that involved with the local
+ * transaction we don't use serverid and userid that are necessary to get
+ * user mapping that is the key of connection cache.
+ */
+bool
+postgresCommitForeignTransaction(FdwXactState *state)
+{
+	PgFdwXactState *rstate;
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+	PGresult	*res;
+
+	entry = GetConnectionState(state->umid, false, false);
+
+	if (!entry->xact_got_connection || !entry->conn)
+		return true;
+
+	rstate = (PgFdwXactState *) palloc0(sizeof(PgFdwXactState));
+	rstate->serverid = state->serverid;
+	rstate->userid = state->userid;
+	rstate->umid = state->umid;
+	rstate->conn = entry;
+	state->fdw_state = (void *)rstate;
+
+	/*
+	 * If abort cleanup previously failed for this connection,
+	 * we can't issue any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	/*
+	 * If there were any errors in subtransactions, and we
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+	entry->have_prep_stmt = false;
+	entry->have_error = false;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/*
+ * The function rollbacks the transactionon foreign server. This
+ * function is called both at the pre-commit phase of the local transaction
+ * when committing and at the end of the local transaction when aborting.
+ * Since we should the connections to the server that involved with the local
+ * transaction we don't use serverid and userid that are necessary to get
+ * user mapping that is the key of connection cache.
+ */
+bool
+postgresRollbackForeignTransaction(FdwXactState *state)
+{
+	PgFdwXactState *rstate = (PgFdwXactState *) state->fdw_state;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (rstate)
+		entry = rstate->conn;
+	else
+		entry = GetConnectionCacheEntry(state->umid);
+
+	/*
+	 * In rollback local transaction, not having connection entry means that
+	 * no transaction started. So we can regard it as success.
+	 */
+	if (!entry->xact_got_connection || !entry->conn)
+		return true;
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is already unsalvageable, do only the cleanup
+	 * and don't touch it further.
+	 */
+	if (entry->changing_xact_state)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return true;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+	else
+	{
+		entry->have_prep_stmt = false;
+		entry->have_error = false;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return !abort_cleanup_failure;
+}
+
+bool
+postgresResolveForeignTransaction(FdwXactState *state, bool is_commit)
+{
+	ConnCacheEntry *entry = NULL;
+	StringInfo	command;
+	bool result = false;
+	PGresult	*res;
+
+	entry = GetConnectionState(state->umid, false, false);
+
+	if (!entry->conn)
+		return false;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 state->fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		/*
+		 * The command failed, raise a warning to log the reason of failure.
+		 * We may not be in a transaction here, so raising error doesn't
+		 * help. Even if we are in a transaction, it would be the resolver
+		 * transaction, which will get aborted on raising error, thus
+		 * delaying resolution of other prepared foreign transactions.
+		 */
+		pgfdw_report_error(LOG, res, entry->conn, false, command->data);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * If we tried to COMMIT/ABORT a prepared transaction and the prepared
+		 * transaction was missing on the foreign server, it was probably
+		 * resolved by some other means. Anyway, it should be considered as resolved.
+		 */
+		result = (sqlstate == ERRCODE_UNDEFINED_OBJECT);
+	}
+	else
+		result = true;
+
+	elog(DEBUG1, "%s prepared foreign transaction on server %u with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 state->serverid,
+		 state->fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->am_participant_of_ac = false;
+
+	/*
+	 * Regardless of the event type, we can now mark ourselves as out of the
+	 * transaction.
+	 */
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 21a2ef5..15dadf4 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,15 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -185,16 +207,19 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                                      List of foreign tables
- Schema |   Table    |  Server   |                   FDW options                    | Description 
---------+------------+-----------+--------------------------------------------------+-------------
- public | ft1        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft2        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft4        | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
- public | ft5        | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft6        | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft_pg_type | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
-(6 rows)
+                                         List of foreign tables
+ Schema |      Table       |  Server   |                   FDW options                    | Description 
+--------+------------------+-----------+--------------------------------------------------+-------------
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
+(9 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8650,3 +8675,345 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+\det+
+                                                 List of foreign tables
+ Schema |      Table       |  Server   |                            FDW options                            | Description 
+--------+------------------+-----------+-------------------------------------------------------------------+-------------
+ public | fpagg_tab_p1     | loopback  | (table_name 'pagg_tab_p1')                                        | 
+ public | fpagg_tab_p2     | loopback  | (table_name 'pagg_tab_p2')                                        | 
+ public | fpagg_tab_p3     | loopback  | (table_name 'pagg_tab_p3')                                        | 
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')                             | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1', use_remote_estimate 'true') | 
+ public | ft3              | loopback  | (table_name 'loct3', use_remote_estimate 'true')                  | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')                             | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type')                  | 
+ public | ftprt1_p1        | loopback  | (table_name 'fprt1_p1', use_remote_estimate 'true')               | 
+ public | ftprt1_p2        | loopback  | (table_name 'fprt1_p2')                                           | 
+ public | ftprt2_p1        | loopback  | (table_name 'fprt2_p1', use_remote_estimate 'true')               | 
+ public | ftprt2_p2        | loopback  | (table_name 'fprt2_p2', use_remote_estimate 'true')               | 
+ public | rem1             | loopback  | (table_name 'loc1')                                               | 
+ public | rem2             | loopback  | (table_name 'loc2')                                               | 
+(19 rows)
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+  srvname  
+-----------
+ loopback
+ loopback2
+ loopback3
+(3 rows)
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO on;
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO ft8_twophase VALUES(3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO "S 1"."T 6" VALUES (4);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  4
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(5);
+INSERT INTO "S 1"."T 6" VALUES (5);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  4
+(1 row)
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(8);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Fails, cannot commit the distributed transaction if 2PC-non-capable
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+ERROR:  cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Disables atomic commit, and success the same case as above.
+SET foreign_twophase_commit TO off;
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+(5 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+(5 rows)
+
+-- Enable atomic commit, again.
+SET foreign_twophase_commit TO on;
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(10);
+INSERT INTO ft8_twophase VALUES(10);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+-- Fails, cannot prepare the transaction if non-supporeted
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(11);
+INSERT INTO ft9_not_twophase VALUES(11);
+PREPARE TRANSACTION 'gx1'; -- error
+ERROR:  cannot prepare a transaction that modified remote tables
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
diff --git a/contrib/postgres_fdw/option.c b/contrib/postgres_fdw/option.c
index 6854f1b..1f45b1c 100644
--- a/contrib/postgres_fdw/option.c
+++ b/contrib/postgres_fdw/option.c
@@ -108,7 +108,8 @@ postgres_fdw_validator(PG_FUNCTION_ARGS)
 		 * Validate option value, when we can do so without any context.
 		 */
 		if (strcmp(def->defname, "use_remote_estimate") == 0 ||
-			strcmp(def->defname, "updatable") == 0)
+			strcmp(def->defname, "updatable") == 0 ||
+			strcmp(def->defname, "two_phase_commit") == 0)
 		{
 			/* these accept only boolean values */
 			(void) defGetBoolean(def);
@@ -177,6 +178,8 @@ InitPgFdwOptions(void)
 		/* fetch_size is available on both server and table */
 		{"fetch_size", ForeignServerRelationId, false},
 		{"fetch_size", ForeignTableRelationId, false},
+		/* two phase commit support */
+		{"two_phase_commit", ForeignServerRelationId, false},
 		{NULL, InvalidOid, false}
 	};
 
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index fd20aa9..1135046 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,8 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "catalog/pg_class.h"
@@ -359,6 +361,7 @@ static void postgresGetForeignUpperPaths(PlannerInfo *root,
 							 RelOptInfo *input_rel,
 							 RelOptInfo *output_rel,
 							 void *extra);
+static bool postgresIsTwoPhaseCommitEnabled(Oid serverid);
 
 /*
  * Helper functions
@@ -452,7 +455,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 				  const PgFdwRelationInfo *fpinfo_o,
 				  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -506,10 +508,29 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+	routine->ResolveForeignTransaction = postgresResolveForeignTransaction;
+	routine->IsTwoPhaseCommitEnabled = postgresIsTwoPhaseCommitEnabled;
+
 	PG_RETURN_POINTER(routine);
 }
 
 /*
+ * postgresIsTwoPhaseCommitEnabled
+ */
+static bool
+postgresIsTwoPhaseCommitEnabled(Oid serverid)
+{
+	ForeignServer	*server = GetForeignServer(serverid);
+
+
+	return server_uses_twophase_commit(server);
+}
+
+/*
  * postgresGetForeignRelSize
  *		Estimate # of rows and width of the result of the scan
  *
@@ -1356,7 +1377,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2411,7 +2432,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2704,7 +2725,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								&retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3321,7 +3342,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4108,7 +4129,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4198,7 +4219,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4421,7 +4442,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
@@ -5803,3 +5824,26 @@ find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel)
 	/* We didn't find any suitable equivalence class expression */
 	return NULL;
 }
+
+/*
+ * server_uses_twophase_commit
+ * Returns true if the foreign server is configured to support 2PC.
+ */
+bool
+server_uses_twophase_commit(ForeignServer *server)
+{
+	ListCell		*lc;
+
+	/* Check the options for two phase compliance */
+	foreach(lc, server->options)
+	{
+		DefElem    *d = (DefElem *) lfirst(lc);
+
+		if (strcmp(d->defname, "two_phase_commit") == 0)
+		{
+			return defGetBoolean(d);
+		}
+	}
+	/* By default a server is not 2PC compliant */
+	return false;
+}
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index 70b538e..3526923 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/relation.h"
@@ -110,12 +111,14 @@ typedef struct PgFdwRelationInfo
 	int			relation_index;
 } PgFdwRelationInfo;
 
+typedef struct ConnCacheEntry ConnCacheEntry;
+
 /* in postgres_fdw.c */
 extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -123,6 +126,11 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 				   bool clear, const char *sql);
+extern bool postgresPrepareForeignTransaction(FdwXactState *state);
+extern bool postgresCommitForeignTransaction(FdwXactState *state);
+extern bool postgresRollbackForeignTransaction(FdwXactState *state);
+extern bool postgresResolveForeignTransaction(FdwXactState *state,
+											  bool is_commit);
 
 /* in option.c */
 extern int ExtractConnectionOptions(List *defelems,
@@ -181,6 +189,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 						List *remote_conds, List *pathkeys, bool is_subquery,
 						List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 88c4cb4..2554c9c 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,19 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -2304,7 +2331,6 @@ SELECT t1.a, t2.b FROM fprt1 t1 INNER JOIN fprt2 t2 ON (t1.a = t2.b) WHERE t1.a
 
 RESET enable_partitionwise_join;
 
-
 -- ===================================================================
 -- test partitionwise aggregates
 -- ===================================================================
@@ -2354,3 +2380,126 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+
+\det+
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO on;
+
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO ft8_twophase VALUES(3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO "S 1"."T 6" VALUES (4);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(5);
+INSERT INTO "S 1"."T 6" VALUES (5);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(8);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Fails, cannot commit the distributed transaction if 2PC-non-capable
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Disables atomic commit, and success the same case as above.
+SET foreign_twophase_commit TO off;
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Enable atomic commit, again.
+SET foreign_twophase_commit TO on;
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(10);
+INSERT INTO ft8_twophase VALUES(10);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Fails, cannot prepare the transaction if non-supporeted
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(11);
+INSERT INTO ft9_not_twophase VALUES(11);
+PREPARE TRANSACTION 'gx1'; -- error
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 54b5e98..f4a9ff5 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -436,6 +436,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
-- 
2.10.5

From 0d59baa5bc23fc51aeebea53a4d644c031f0db0b Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:48:08 +0900
Subject: [PATCH v20 4/4] Add regression tests for atomic commit.

---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index daf79a0..71c8b9d 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000..a23f120
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port', two_phase_commit 'on');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port', two_phase_commit 'on');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_prepared_fdw_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 6890678..d1b181a 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2286,9 +2286,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2303,7 +2306,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.10.5

From 4c078f27add47d8b915524323484bc3c2e51a3e5 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:44:42 +0900
Subject: [PATCH v20 2/4] Support atomic commit among multiple foreign servers.

---
 doc/src/sgml/catalogs.sgml                    |   97 +
 doc/src/sgml/config.sgml                      |  143 +-
 doc/src/sgml/distributed-transaction.sgml     |  157 ++
 doc/src/sgml/fdwhandler.sgml                  |  203 ++
 doc/src/sgml/filelist.sgml                    |    1 +
 doc/src/sgml/func.sgml                        |   51 +
 doc/src/sgml/monitoring.sgml                  |   56 +
 doc/src/sgml/postgres.sgml                    |    1 +
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/fdwxact.c          | 2678 +++++++++++++++++++++++++
 src/backend/access/fdwxact/fdwxact_launcher.c |  641 ++++++
 src/backend/access/fdwxact/fdwxact_resolver.c |  331 +++
 src/backend/access/heap/heapam.c              |   12 -
 src/backend/access/rmgrdesc/Makefile          |    8 +-
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   65 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/Makefile           |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   32 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/foreigncmds.c            |   23 +
 src/backend/executor/execPartition.c          |    4 +
 src/backend/executor/nodeForeignscan.c        |    8 +
 src/backend/executor/nodeModifyTable.c        |   24 +
 src/backend/foreign/foreign.c                 |   43 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   18 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    2 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   80 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  147 ++
 src/include/access/fdwxact_launcher.h         |   32 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   52 +
 src/include/access/resolver_internal.h        |   67 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   23 +
 src/include/foreign/fdwapi.h                  |   18 +-
 src/include/foreign/foreign.h                 |    2 +-
 src/include/pgstat.h                          |    8 +-
 src/include/storage/proc.h                    |   10 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    2 +
 src/test/regress/expected/rules.out           |   12 +
 62 files changed, 5283 insertions(+), 40 deletions(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100755 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/fdwxact_launcher.c
 create mode 100644 src/backend/access/fdwxact/fdwxact_resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 6d6fbec..9d99cdc 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9622,6 +9622,103 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-prepared-fdw-xacts">
+  <title><structname>pg_prepared_fdw_xacts</structname></title>
+
+  <indexterm zone="view-pg-prepared-fdw-xacts">
+   <primary>pg_prepared_fdw_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_prepared_fdw_xacts</structname> displays
+   information about foreign transactions that are currently prepared on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="fdw-transaction-managements"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_prepared_xacts</structname> contains one row per prepared
+   foreign transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_prepared_fdw_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>transaction</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Transaction id that this foreign transaction associates with
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server that this foreign server is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction: <literal>prepared</literal>, <literal>committing</literal>, <literal>aborting</literal> or <literal>unknown</literal>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7554cba..557b3f2 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3611,7 +3611,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -7827,6 +7826,148 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophase_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign transaction
+         to be resolved before the command returns a "success" indication to the client.
+         Valid values are <literal>required</literal>, <literal>prefer</literal> and
+         <literal>disabled</literal>. The default setting is <literal>disabled</literal>.
+         When <literal>disabled</literal>, there can be risk of database consistency among
+         distributed transaction if some foreign server crashes during committing the
+         distributed transaction. When set to <literal>required</literal> the distributed
+         transaction requires that all written servers can use two-phase commit protocol.
+         That is, the transaction fails if any of servers returns <literal>false</literal>
+         from <function>IsTwoPhaseCommitEnabled</function> or does not support transaction
+         management callback routines(described in
+         <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction requires
+         two-phase commit protocol where available but without failing when it is not
+         available.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero value to
+         set this parameter either <literal>required</literal> or <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one transaction
+         is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism.  You should set this value to
+         zero only if you set <varname>max_foreign_transaction_resolvers</varname> as
+         much as databases you have. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000..5143499
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,157 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction Management</title>
+
+ <para>
+  This chapter explains what distributed transaction management is, and how it can be configured
+  in PostgreSQL.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit is an operation that applies a set of changes as a single operation
+   globally. <productname>PostgreSQL</productname> provides a way to perform a transaction
+   with foreign resources using <literal>Foreign Data Wrapper</literal>. Using the
+   <productname>PostgreSQL</productname>'s atomic commit ensures that all changes
+   on foreign servers end in either commit or rollback using the transaction callback
+   routines (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs Two-phase commit protocol, which is a
+    type of atomic commitment protocol (ACP). Using Two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed transaction manager
+    prepares all transaction on the foreign servers if two-phase commit is required.
+    Two-phase commit is required only if the transaction modifies data on two or more
+    servers including the local server itself and user requests it by
+    <xref linkend="guc-foreign-twophase-commit"/>. If all preparations on foreign servers
+    got successful go to the next step. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes over rollback, then rollback all transactions
+    on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the transaction
+    locally. Any failure happens in this step <productname>PostgreSQL</productname> changes
+    over rollback, then rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign Transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Foreign Transaction Resolution</title>
+
+   <para>
+    Foreign transaction resolutions are performed by foreign transaction resolver process.
+    They commit all prepared transaction on foreign servers if the coordinator received
+    an agreement message from all foreign server during the first step. On the other hand,
+    if any foreign server failed to prepare the transaction, it rollbacks all prepared
+    transactions.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions on one
+    database of the coordinator side. On failure during resolution, they retries to
+    resolve after <varname>foreign_transaction_resolution_interval</varname>.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>In-doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit or rollback
+    using two-phase commit protocol. However, if the second phase fails for whatever reason
+    the transaction becomes in-doubt. The transactions becomes in-doubt in the following
+    situations:
+
+   <itemizedlist>
+    <listitem>
+     <para>
+      A local <productname>PostgreSQL</productname> server crashes during atomic commit
+      operation.
+     </para>
+    </listitem>
+
+    <listitem>
+     <para>
+      A local <productname>PostgreSQL</productname> server got a cancellation by user during
+      atomic commit.
+     </para>
+    </listitem>
+   </itemizedlist>
+
+   In-doubt transactions are automatically handled by foreign transaction resolver process
+   when there is no online transaction requesting resolutions.
+   <function>pg_resolve_fdw_xact</function> provides a way to resolve transactions on foreign
+   servers manually that participated the distributed transaction manually.
+   </para>
+
+   <para>
+    The atomic commit operation is crash-safe. The being processed foreign transactions at
+    crash are processed by a foreign transaction resolvers as an in-doubt transaction
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-fdwxact-resolver-view"><literal>pg_stat_fdwxact_resolver</literal></link>
+    view. This view contains one row for every foreign Transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 4ce88dd..3da13c9 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1390,6 +1390,118 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     If an FDW wishes to support <firstterm>atomic commit</firstterm>
+     (as described in <xref linkend="fdw-transaction-managements"/>), it must call the
+     registrasaction function <function>FdwXactRegisterForeignTransaction</function>
+     and provide the following callback functions:
+    </para>
+
+    <para>
+<programlisting>
+bool
+PrepareForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if atomic commit is required.
+    Returning <literal>true</literal> means that preparing the foreign
+    transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Commit the not-prepared transaction on the foreign server.
+    This function is called at the pre-commit phase of local
+    transaction if atomic commit is not required. The atomic
+    commit is not required either when we modified data on
+    only one server including the local server or when userdoesn't
+    request atomic commit by <xref linkend="guc-foreign-twophase-commit"/>.
+    Returning <literal>true</literal> means that commit the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Rollback a not-prepared transaction on the foreign server.
+    This function is called at the end of local transaction after
+    rollbacked locally either when user requested rollback or when
+    any error occurs during the transaction. This function could
+    be called recursively if any error occurs during rollback the
+    foreign transaction for whatever reason. You need to track
+    recursion and prevent this function from being called infinitely.
+    Returning <literal>true</literal> means that rollback the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+ResolvePreparedForeignTransaction(FdwXactResolveState *state,
+                                  bool is_commit);
+</programlisting>
+    Commit or rollback the prepared transaction on the foreign server.
+    When <varname>is_commit</varname> is true, it indicates that the foreign
+    transaction should be committed. Otherwise the foreign transaction should
+    be aborted.
+    This function normally is called by the foreign transaction resolver
+    process but can also be called by <function>pg_resovle_fdw_xacts</function>
+    function. In the resolver process, this function is called either
+    when a backend requests the resolver process to resolve a distributed
+    transaction after prepared, or when a database has dangling
+    transactions. Returning <literal>true</literal> means that resolving
+    the foreign transaction got successful.
+    In abort case, please note that the prepared transaction identified
+    by <varname>state->fdwxact_id</varname> might not exist on the foreign
+    server. If you failed to resolve the foreign transaction due to undefined
+    object error (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) you should
+   regards it as success and return <literal>true</literal>.
+    </para>
+    <para>
+<programlisting>
+bool
+IsTwoPhaseCommitEnabled(Oid serverid);
+</programlisting>
+    Return <literal>true</literal> if the foreign server identified by
+    <literal>serverid</literal> is capable of two-phase commit protocol.
+    This function is called at commit time once.
+    Return <literal>false</literal> indicates that the current transaction
+    cannot use atomic commit even if atomic commit is requested by user.
+    </para>
+
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1835,4 +1947,95 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+    <title>Transaction managements for Foreign Data Wrappers</title>
+
+    <para>
+     <productname>PostgreSQL</productname> foreign transaction manager
+     allows FDWs to read and write data on foreign server within a transaction while
+     maintaining atomicity of the foreign data (aka atomic commit). Using
+     atomic commit, it guarantees that a distributed transaction is committed
+     or rollbacked on all participants foreign
+     server.  To achieve atomic commit, <productname>PostgreSQL</productname>
+     employees two-phase commit protocol, which is a type of atomic commitment
+     protocol. Every FDW that wish to support atomic commit
+     is required to support the transaction management callback routines:
+     <function>PrepareForeignTransaction</function>,
+     <function>CommitForeignTransaction</function>,
+     <function>RollbackForeignTransaction</function>,
+     <function>ResolveForeignTransaction</function>,
+     <function>IsTwoPhaseCommitEnabled</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+     Tranasction of foreign server that supports these callback routines is
+     managed by <productname>PostgreSQL</productname>'s distributed  transaction
+     manager. Each transaction management callbacks are called at appropriate time.
+    </para>
+
+    <para>
+     The information in <literal>FdwXactState</literal> can be used to identify
+     foreign servers. <literal>state-&gt;fdw_state</literal> is a <type>void</type>
+     pointer that is available for FDW transaction functions to store Information
+     relevant to the particular foreign server.  It is useful for passing
+     information forward from <function>PrepareForeignTransaction</function> and/or
+     <function>CommitTransaciton</function> to
+     <function>RollbackForeignTransaction</function>, there by avoiding recalculation.
+     Note that since <function>ResolveForeignTransaction</function> is called
+     idependently from these callback routines, the information is not passed to
+     <function>ResolverForeignTransaction</function>.
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling <function>PrepareForeignTransaction</function>
+     if two-phase commit protocol is required. Two-phase commit is required only if
+     the transaction modified data on more than one servers including the local
+     server itself and user requests atomic commit. <productname>PostgreSQL</productname>
+     can commit locally and go to the next step if and only if all preparing foreign
+     transactions got successful. If two-phase commit is not required, the foreign
+     transaction manager commits each transaction calling
+     <function>CommitForeignTransaction</function> and then commit locally.
+     If any failure happens or user requests to cancel during the pre-commit phase
+     the distributed Transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function> for not-prepared foreign
+     servers, and then rollback locally. The prepared foreign servers are rollbacked
+     by a foreign transaction resolver process.
+    </para>
+
+    <para>
+     Once committed locally, the distributed transaction must be committed. The
+     prepared foreign transaction will be committed by foreign transaction resolver
+     process.
+    </para>
+
+    <para>
+     When two-phase commit is required, after committed locally, the transaction
+     commit will wait for all prepared foreign transaction to be committed before
+     completetion. One foreign transaction resolver process is responsible for
+     foreign transaction resolution on a database.
+     <function>ResolverForeignTransaction</function> is called by the foreign
+     transaction resolver process when resolution.
+     <function>ResolveForeignTransaction</function> is also be called
+     when user executes <function>pg_resovle_fdw_xact</function> function.
+    </para>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 48ac14a..38d6fcb 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -49,6 +49,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 5193df3..8bb251e 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -20806,6 +20806,57 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-fdw-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_fdw_xacts</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_fdw_xacts</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function search for foreign transaction
+        matching the arguments and resolves then. This function won't resolve
+        a foreign transaction which is in progress, or one that is locked by some
+        other backend.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_fdw_xact</function>
+        except it remove foreign transaction entry without resolving.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 0484cfa..6b2aa6f 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -332,6 +332,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_fdw_xact_resolver</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-fdwxact-resolver-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1194,6 +1202,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
          <entry><literal>LogicalLauncherMain</literal></entry>
          <entry>Waiting in main loop of logical launcher process.</entry>
         </row>
@@ -1405,6 +1421,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
+        <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
          <entry>Waiting during base backup when throttling activity.</entry>
@@ -2214,6 +2234,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-fdwxact-resolver-view" xreflabel="pg_stat_fdw_xact_resolver">
+   <title><structname>pg_stat_fdw_xact_resolver</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index 0070603..c10e21f 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -164,6 +164,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index bd93a6a..4a1ebdc 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  tablesample transam
+			  tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000..9ddbb14
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o fdwxact_resolver.o fdwxact_launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100755
index 0000000..1f270db
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2678 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL distributed transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers automically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * When a foreign data wrapper starts transaction on a foreign server that
+ * is capable of two-phase commit protocol, foreign data wrappers registers
+ * the foreign transaction using function FdwXactRegisterForeignTransaction()
+ * in order to participate to a group for atomic commit. Participants are
+ * identified by oid of foreign server and user. When the foreign transaction
+ * begins to modify data the executor marks it as modified using
+ * FdwXactMarkForeignTransactionModified().
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. After committing or rolling back locally, we
+ * notify the resolver process and tell it to commit or roll back those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail when an ERROR
+ * after already committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * waiters each time we receive a request. We have two queues: the active
+ * queue and the retry queue. The backend is inserted to the active queue at
+ * first, and then it is moved to the retry queue by the resolver process if
+ * the resolution fails. The backends in the retry queue are processed at
+ * interval of foreign_transaction_resolution_retry_interval.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or more
+ * servers including itself. In other case, all foreign transactions are
+ * committed during pre-commit.
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. dangling
+ * transaction). Dangling transactions are processed by the resolve process
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * 	* On PREPARE redo we add the foreign transaction to FdwXactCtl->fdw_xacts.
+ *	  We set fdw_xact->inredo to true for such entries.
+ *	* On Checkpoint redo, we iterate through FdwXactCtl->fdw_xacts entries that
+ *	  have set fdw_xact->inredo true and are behind the redo_horizon. We save
+ *    them to disk and then set fdw_xact->ondisk to true.
+ *	* On COMMIT and ABORT we delete the entry from FdwXactCtl->fdw_xacts.
+ *	  If fdw_xact->ondisk is true, we delete the corresponding file from
+ *	  the disk as well.
+ *  * RecoverFdwXacts loads all foreign transaction entries from disk into
+ *    memory at server startup.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Is atomic commit requested by user? */
+#define IsAtomicCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+#define IsAtomicCommitRequested() \
+	(IsAtomicCommitEnabled() && \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED))
+
+#define FDW_XACT_ACTION_COMMIT	 		0x01
+#define FDW_XACT_ACTION_TWOPHASE_COMMIT 0x02
+
+/* Structure to bundle the foreign transaction participant */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in global entry. NULL if
+	 * this foreign transaction is registered but not inserted
+	 * yet.
+	 */
+	FdwXact		fdw_xact;
+	char		*fdw_xact_id;
+
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	bool		modified;					/* true if modified the data on server */
+	bool		twophase_commit_enabled;	/* true if the server can execute
+											 * two-phase commit protocol */
+	void			*fdw_state;				/* fdw-private state */
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact;
+	CommitForeignTransaction_function	commit_foreign_xact;
+	RollbackForeignTransaction_function	rollback_foreign_xact;
+	IsTwoPhaseCommitEnabled_function	is_twophase_commit_enabled;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit.
+ * This list has only foreign servers that support atomic commit FDW
+ * API regardless of their configuration.
+ */
+static List *FdwXactAtomicCommitParticipants = NIL;
+static bool FdwXactAtomicCommitReady = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDW_XACTS_DIR "pg_fdw_xact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDW_XACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDW_XACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+static void FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, bool modified);
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactCommitForeignTransaction(FdwXactParticipant *fdw_part);
+static bool FdwXactResolveForeignTransaction(FdwXactState *state, FdwXact fdwxact,
+											 int elevel);
+static void FdwXactComputeRequiredXmin(void);
+static bool FdwXactAtomicCommitRequired(void);
+static void FdwXactQueueInsert(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid, bool give_warnings);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							  bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid, Oid serverid,
+								  Oid userid, XLogRecPtr insert_start_lsn,
+								  bool give_warnings);
+static List *get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						   bool need_lock);
+static FdwXact get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								bool need_lock);
+static FdwXact get_all_fdw_xacts(int *length);
+static FdwXact insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							   Oid umid, char *fdw_xact_id);
+static char *generate_fdw_xact_identifier(TransactionId xid, Oid serverid, Oid userid);
+static void remove_fdw_xact(FdwXact fdw_xact);
+static FdwXactState *create_fdw_xact_state(void);
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+int foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+/*
+ * Remember accessed foreign server. This function is called by executor when
+ * it begins to access foreign server. If FDW of the foreign server supports
+ * atomic commit API, it is registered as a transaction participant of distributed
+ * transaction.
+ */
+void
+FdwXactMarkForeignServerAccessed(Relation rel, int flags, bool modified)
+{
+	FdwRoutine			*fdwroutine;
+	ListCell   			*lc;
+	Oid					serverid;
+	Oid					userid;
+
+	/* Quick return if atomic commit is not enabled */
+	if (!IsAtomicCommitEnabled())
+		return;
+
+	/* Do nothing in EXPLAIN (no ANALYZE) case */
+	if (flags && EXEC_FLAG_EXPLAIN_ONLY)
+		return;
+
+	serverid = GetForeignServerIdByRelId(RelationGetRelid(rel));
+	fdwroutine  = GetFdwRoutineByRelId(RelationGetRelid(rel));
+
+	/*
+	 * If the being modified foreign server doesn't have the atomic commit API
+	 * we don't manage the foreign transaction in the distributed transaction
+	 * manager.
+	 */
+	if (fdwroutine->IsTwoPhaseCommitEnabled == NULL)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		return;
+	}
+
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	foreach(lc, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	FdwXactRegisterForeignTransaction(serverid, userid, modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign server identified by given server id must support atomic
+ * commit APIs. Registered foreign transaction are managed by foreign
+ * transaction manager until the end of the transaction.
+ */
+static void
+FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant	*fdw_part;
+	ForeignServer 		*foreign_server;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	FdwRoutine			*fdw_routine;
+	MemoryContext		old_ctx;
+	char				*fdwxact_id;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	/*
+	 * Participants information is needed at the end of a transaction, where
+	 * system cache are not available. Save it in TopTransactionContext
+	 * beforehand so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	foreign_server = GetForeignServer(serverid);
+	fdw = GetForeignDataWrapper(foreign_server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+	user_mapping = GetUserMapping(userid, serverid);
+
+	/* Make sure that the FDW has transaction handlers */
+	if (!fdw_routine->PrepareForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function provided for preparing foreign transaction for FDW %s",
+						fdw->fdwname)));
+	if (!fdw_routine->CommitForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to commit a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+	if (!fdw_routine->RollbackForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to rollback a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+
+	/* Generate an unique identifier */
+	if (fdw_routine->GetPrepareId)
+	{
+		char *id;
+		int fdwxact_id_len = 0;
+
+		id = fdw_routine->GetPrepareId(GetTopTransactionId(),
+											   foreign_server->serverid,
+											   user_mapping->userid,
+											   &fdwxact_id_len);
+
+		if (!id)
+			ereport(ERROR,
+					(errcode(ERRCODE_UNDEFINED_OBJECT),
+					 (errmsg("foreign transaction identifier is not provided"))));
+
+		/* Check length of foreign transaction identifier */
+		id[fdwxact_id_len] = '\0';
+		if (fdwxact_id_len > NAMEDATALEN)
+			ereport(ERROR,
+					(errcode(ERRCODE_NAME_TOO_LONG),
+					 errmsg("foreign transaction identifer \"%s\" is too long",
+							id),
+					 errdetail("foreign transaction identifier must be less than %d characters.",
+							   NAMEDATALEN)));
+
+		fdwxact_id = pstrdup(id);
+	}
+	else
+		fdwxact_id = generate_fdw_xact_identifier(GetTopTransactionId(),
+												  serverid, userid);
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdw_xact_id = fdwxact_id;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdw_xact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->twophase_commit_enabled = true; /* by default, will be changed at pre-commit phase */
+	fdw_part->fdw_state = NULL;
+	fdw_part->prepare_foreign_xact = fdw_routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact = fdw_routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact = fdw_routine->RollbackForeignTransaction;
+	fdw_part->is_twophase_commit_enabled = fdw_routine->IsTwoPhaseCommitEnabled;
+
+	/* Add this foreign transaction to the participants list */
+	FdwXactAtomicCommitParticipants = lappend(FdwXactAtomicCommitParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+
+	return;
+}
+
+/*
+ * FdwXactShmemSize
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdw_xacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * FdwXactShmemInit
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdw_xacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->freeFdwXacts = NULL;
+		FdwXactCtl->numFdwXacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdw_xacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdw_xacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdw_xacts[cnt].status = FDW_XACT_INITIAL;
+			fdw_xacts[cnt].fxact_free_next = FdwXactCtl->freeFdwXacts;
+			FdwXactCtl->freeFdwXacts = &fdw_xacts[cnt];
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * PreCommit_FdwXacts
+ *
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_atomic_commit;
+	ListCell	*lc;
+	ListCell	*next;
+	ListCell	*prev = NULL;
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsAtomicCommitRequested())
+		return;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	need_atomic_commit = FdwXactAtomicCommitRequired();
+
+	/*
+	 * If 'require' case, we require all modified server have to be capable of
+	 * two-phase commit protocol.
+	 */
+	if (need_atomic_commit &&
+		foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	/*
+	 * Commit transactions on foreign servers.
+	 *
+	 * Committed transactions are removed from FdwXactAtomicCommitParticipants
+	 * so that the later preparation can process only servers that requires to be commit
+	 * using two-phase commit protocol.
+	 */
+	for (lc = list_head(FdwXactAtomicCommitParticipants); lc != NULL; lc = next)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool can_commit = false;
+
+		next = lnext(lc);
+
+		if (!need_atomic_commit || !fdw_part->modified)
+		{
+			/*
+			 * We can commit not-modified servers and when the atomic commit is not
+			 * required.
+			 */
+			can_commit = true;
+		}
+		else if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER &&
+				 !fdw_part->twophase_commit_enabled)
+		{
+			/* Also in 'prefer' case, non-2pc-capable servers can be committed */
+			can_commit = true;
+		}
+
+		if (can_commit)
+		{
+			/* Commit the foreign transaction */
+			FdwXactCommitForeignTransaction(fdw_part);
+
+			/* Delete it from the participant list */
+			FdwXactAtomicCommitParticipants =
+				list_delete_cell(FdwXactAtomicCommitParticipants, lc, prev);
+		}
+
+		prev = lc;
+	}
+
+	/*
+	 * If only one participant of all participants is modified, we can commit it.
+	 * This can avoid to use two-phase commit for only one server in the 'prefer' case
+	 * where the transaction has one 2pc-capable modified server and some modified
+	 * servers.
+	 */
+	if (list_length(FdwXactAtomicCommitParticipants) == 1 &&
+		(MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) == 0)
+	{
+		Assert(foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER);
+		FdwXactCommitForeignTransaction(linitial(FdwXactAtomicCommitParticipants));
+		list_free(FdwXactAtomicCommitParticipants);
+		return;
+	}
+
+	FdwXactPrepareForeignTransactions();
+	/* keep FdwXactparticipantsForAC until the end of transaction */
+}
+
+/*
+ * FdwXactPrepareForeignTransactions
+ *
+ * Prepare all foreign transaction participants.  This function creates a prepared
+ * participants chain each time when we prepared a foreign transaction. The prepared
+ * participants chain is used to access all participants of distributed transaction
+ * quickly. If any one of them fails to prepare, we change over aborts.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	FdwXactState *state;
+	ListCell   *lcell;
+	FdwXact		prev_fdwxact = NULL;
+
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	state = create_fdw_xact_state();
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXact		fdwxact;
+
+		/*
+		 * Insert the foreign transaction entry. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(GetTopTransactionId(), fdw_part);
+
+		state->serverid = fdw_part->server->serverid;
+		state->userid = fdw_part->usermapping->userid;
+		state->umid = fdw_part->usermapping->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		/*
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal). During abort processing,
+		 * we might try to resolve a never-prepared transaction, and get an error.
+		 * This is fine as long as the FDW provides us unique prepared transaction
+		 * identifiers.
+		 */
+		if (!fdw_part->prepare_foreign_xact(state))
+		{
+			/* Failed to prepare, change over aborts */
+			ereport(ERROR,
+					(errmsg("could not prepare transaction on foreign server %s",
+							fdw_part->server->servername)));
+		}
+
+		/* Keep fdw_state until end of transaction */
+		fdw_part->fdw_state = state->fdw_state;
+
+		/* Preparation is success, update its status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdw_part->fdw_xact->status = FDW_XACT_PREPARED;
+		fdw_part->fdw_xact = fdwxact;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Create a prepared participants chain, which is link-ed FdwXact entries
+		 * involving with this transaction.
+		 */
+		if (prev_fdwxact)
+		{
+			/* Append others to the tail */
+			Assert(fdwxact->fxact_next == NULL);
+			prev_fdwxact->fxact_next = fdwxact;
+		}
+	}
+}
+
+/*
+ * Commit the given foreign transaction.
+ */
+void
+FdwXactCommitForeignTransaction(FdwXactParticipant *fdw_part)
+{
+	FdwXactState *state;
+
+	state = create_fdw_xact_state();
+	state->serverid = fdw_part->server->serverid;
+	state->userid = fdw_part->usermapping->userid;
+	state->umid = fdw_part->usermapping->umid;
+	fdw_part->fdw_state = (void *) state;
+
+	if (!fdw_part->commit_foreign_xact(state))
+		ereport(ERROR,
+				(errmsg("could not commit foreign transaction on server %s",
+						fdw_part->server->servername)));
+}
+
+/*
+ * FdwXactInsertFdwXactEntry
+ *
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and will
+ * be persisted to the disk under pg_fdw_xact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fxact;
+	FdwXactOnDiskData	*fxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact = insert_fdw_xact(MyDatabaseId, xid, fdw_part->server->serverid,
+							fdw_part->usermapping->userid,
+							fdw_part->usermapping->umid, fdw_part->fdw_xact_id);
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->held_by = MyBackendId;
+	fdw_part->fdw_xact = fxact;
+	LWLockRelease(FdwXactLock);
+
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdw_xact_id);
+	data_len = data_len + strlen(fdw_part->fdw_xact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fxact_file_data->dbid = MyDatabaseId;
+	fxact_file_data->local_xid = xid;
+	fxact_file_data->serverid = fdw_part->server->serverid;
+	fxact_file_data->userid = fdw_part->usermapping->userid;
+	fxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fxact_file_data->fdw_xact_id, fdw_part->fdw_xact_id,
+		   strlen(fdw_part->fdw_xact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fxact_file_data, data_len);
+	fxact->insert_end_lsn = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_INSERT);
+	XLogFlush(fxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact->valid = true;
+	LWLockRelease(FdwXactLock);
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fxact_file_data);
+	return fxact;
+}
+
+/*
+ * insert_fdw_xact
+ *
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				Oid umid, char *fdw_xact_id)
+{
+	int i;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		fxact = FdwXactCtl->fdw_xacts[i];
+		if (fxact->dbid == dbid &&
+			fxact->local_xid == xid &&
+			fxact->serverid == serverid &&
+			fxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->freeFdwXacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fxact = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fxact->fxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->numFdwXacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts++] = fxact;
+
+	fxact->held_by = InvalidBackendId;
+	fxact->dbid = dbid;
+	fxact->local_xid = xid;
+	fxact->serverid = serverid;
+	fxact->userid = userid;
+	fxact->umid = umid;
+	fxact->insert_start_lsn = InvalidXLogRecPtr;
+	fxact->insert_end_lsn = InvalidXLogRecPtr;
+	fxact->valid = false;
+	fxact->ondisk = false;
+	fxact->inredo = false;
+	memcpy(fxact->fdw_xact_id, fdw_xact_id, strlen(fdw_xact_id) + 1);
+
+	return fxact;
+}
+
+/*
+ * remove_fdw_xact
+ *
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdw_xact(FdwXact fdw_xact)
+{
+	int			cnt;
+
+	Assert(fdw_xact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Search the slot where this entry resided */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		if (FdwXactCtl->fdw_xacts[cnt] == fdw_xact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (cnt >= FdwXactCtl->numFdwXacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdw_xact->local_xid, fdw_xact->serverid, fdw_xact->userid)));
+
+	/* Remove the entry from active array */
+	FdwXactCtl->numFdwXacts--;
+	FdwXactCtl->fdw_xacts[cnt] = FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts];
+
+	/* Put it back into free list */
+	fdw_xact->fxact_free_next = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fdw_xact;
+
+	/* Reset informations */
+	fdw_xact->status = FDW_XACT_INITIAL;
+	fdw_xact->held_by = InvalidBackendId;
+	fdw_xact->fxact_next = NULL;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdw_xact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdw_xact->serverid;
+		record.dbid = fdw_xact->dbid;
+		record.xid = fdw_xact->local_xid;
+		record.userid = fdw_xact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdw_xact_remove));
+		recptr = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true and set FdwXactAtomicCommitReady to true if we require atomic commit.
+ * It is required if the transaction modified data on two or more servers including
+ * local node itself. This function also checks for each server if two-phase commit
+ * is enabled or not.
+ */
+static bool
+FdwXactAtomicCommitRequired(void)
+{
+	ListCell*	lc;
+	int			nserverswritten = 0;
+
+	if (!IsAtomicCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		/* Check if the foreign server is capable of two-phase commit protocol */
+		if (fdw_part->is_twophase_commit_enabled(fdw_part->server->serverid))
+			fdw_part->twophase_commit_enabled = true;
+		else if (fdw_part->modified)
+			MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	/* Atomic commit is required if we modified data on two or more participants */
+	if (nserverswritten <= 1)
+		return false;
+
+	FdwXactAtomicCommitReady = true;
+	return true;
+}
+
+bool
+FdwXactIsAtomicCommitReady(void)
+{
+	return FdwXactAtomicCommitReady;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * ForgetAllFdwXactParticipants
+ *
+ * Reset all the foreign transaction entries that this backend registered.
+ * If the foreign transaction has the corresponding FdwXact entry, resetting
+ * the held_by field means to leave that entry in unresolved state. If we
+ * leaves any entries, we update the oldest xmin of unresolved transaction
+ * so that transaction status of dangling transaction are not truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_lefts = 0;
+
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(cell, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+
+		/* Skip if didn't register FdwXact entry yet */
+		if (fdw_part->fdw_xact == NULL)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in
+		 * FdwXactAtomicCommitParticipants could be used by other backend before we
+		 * forget in case where the resolver process removes the FdwXact entry
+		 * and other backend reuses it before we forget. So we need to check
+		 * if the entries are still associated with the transaction.
+		 */
+		if (fdw_part->fdw_xact->held_by == MyBackendId)
+		{
+			fdw_part->fdw_xact->held_by = InvalidBackendId;
+			n_lefts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Update the oldest local transaction of unresolved distributed
+	 * transaction if we leaved any FdwXact entries.
+	 */
+	if (n_lefts > 0)
+		FdwXactComputeRequiredXmin();
+
+	FdwXactAtomicCommitParticipants = NIL;
+}
+
+/*
+ * AtProcExit_FdwXact
+ *
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+/*
+ * Wait for foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDW_XACT_NOT_WAITING and then change
+ * that state to FDW_XACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDW_XACT_WAIT_COMPLETE once foreign transactions are resolved.
+ * This backend then resets its state to FDW_XACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue and changes the state to FDW_XACT_WAITING_RETRY.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+	ListCell	*lc;
+	List		*fdwxact_participants = NIL;
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsAtomicCommitRequested())
+		return;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDW_XACT_NOT_WAITING);
+
+	if (FdwXactAtomicCommitParticipants != NIL)
+	{
+		/*
+		 * If we're waiting for foreign transactions to be resolved that
+		 * we've prepared just before, use the participants list.
+		 */
+		Assert(MyPgXact->xid == wait_xid);
+		fdwxact_participants = FdwXactAtomicCommitParticipants;
+	}
+	else
+	{
+		/*
+		 * Get participants list from the global array. This is required (1)
+		 * when we're waiting for foreign transactions to be resolved that
+		 * is part of a local prepared transaction that is marked as prepared
+		 * during running, or (2) when we resolve the PREPARE'd distributed
+		 * transaction after restart.
+		 */
+		fdwxact_participants = get_fdw_xacts(MyDatabaseId, wait_xid,
+											 InvalidOid, InvalidOid, true);
+	}
+
+	/* Exit if we found no foreign transaction to resolve */
+	if (fdwxact_participants == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(lc, fdwxact_participants)
+	{
+		FdwXact fdw_xact = (FdwXact) lfirst(lc);
+
+		/* Don't overwrite status if fate has been determined */
+		if (fdw_xact->status == FDW_XACT_PREPARED)
+			fdw_xact->status = (is_commit ?
+								FDW_XACT_COMMITTING_PREPARED :
+								FDW_XACT_ABORTING_PREPARED);
+	}
+
+	/* Set backend status and enqueue itself to the active queue*/
+	MyProc->fdwXactState = FDW_XACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	FdwXactQueueInsert();
+	LWLockRelease(FdwXactLock);
+
+	/* Launch a resolver process if not yet, or wake it up */
+	fdwxact_maybe_launch_resolver(false);
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDW_XACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDW_XACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDW_XACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+
+	/*
+	 * Forget the list of locked entries, also means that the entries
+	 * that could not resolved are remained as dangling transactions.
+	 */
+	ForgetAllFdwXactParticipants();
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Acquire FdwXactLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Insert MyProc into the tail of FdwXactActiveQueue.
+ */
+static void
+FdwXactQueueInsert(void)
+{
+	SHMQueueInsertBefore(&(FdwXactRslvCtl->FdwXactActiveQueue),
+						 &(MyProc->fdwXactLinks));
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * Resolve one distributed transaction. The target distributed transaction
+ * is fetched from either the active queue or the retry queue and its participants
+ * are fetched from either the global array.
+ *
+ * Release the waiter and return true if we resolved the all of the foreign
+ * transaction participants. On failure, we move the FdwXactLinks entry to the
+ * retry queue from the active queue, and raise an error and exit.
+ */
+bool
+FdwXactResolveDistributedTransaction(Oid dbid, bool is_active)
+{
+	FdwXactState	*state;
+	ListCell		*lc;
+	ListCell		*next;
+	PGPROC			*waiter = NULL;
+	List			*participants;
+	SHM_QUEUE		*target_queue;
+
+	if (is_active)
+		target_queue = &(FdwXactRslvCtl->FdwXactActiveQueue);
+	else
+		target_queue = &(FdwXactRslvCtl->FdwXactRetryQueue);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/* Fetch a waiter from beginning of the queue */
+	while ((waiter = (PGPROC *) SHMQueueNext(target_queue, target_queue,
+											 offsetof(PGPROC, fdwXactLinks))) != NULL)
+	{
+		/* Found a waiter */
+		if (waiter->databaseId == dbid)
+			break;
+	}
+
+	/* If no waiter, there is no job */
+	if (!waiter)
+	{
+		LWLockRelease(FdwXactLock);
+		return false;
+	}
+
+	Assert(TransactionIdIsValid(waiter->fdwXactWaitXid));
+
+	state = create_fdw_xact_state();
+	participants = get_fdw_xacts(dbid, waiter->fdwXactWaitXid, InvalidOid,
+								 InvalidOid, false);
+	LWLockRelease(FdwXactLock);
+
+	/* Resolve all foreign transactions one by one */
+	for (lc = list_head(participants); lc != NULL; lc = next)
+	{
+		FdwXact fdwxact = (FdwXact) lfirst(lc);
+
+		CHECK_FOR_INTERRUPTS();
+
+		next = lnext(lc);
+
+		state->serverid = fdwxact->serverid;
+		state->userid = fdwxact->userid;
+		state->umid = fdwxact->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		PG_TRY();
+		{
+			FdwXactResolveForeignTransaction(state, fdwxact, ERROR);
+		}
+		PG_CATCH();
+		{
+			/* Re-insert the waiter to the retry queue */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDW_XACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				SHMQueueInsertBefore(&(FdwXactRslvCtl->FdwXactRetryQueue),
+									 &(waiter->fdwXactLinks));
+				waiter->fdwXactState = FDW_XACT_WAITING_RETRY;
+			}
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved a foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+	}
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter
+	 * could already be detached if user cancelled to wait before
+	 * resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId	wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDW_XACT_WAIT_COMPLETE;
+
+		/* Wake up the waiter only when we have set state and removed from queue */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc xid %u", wait_xid);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	return true;
+}
+
+/*
+ * Resolve all dangling foreign transactions on the given database. Get
+ * all dangling foreign transactions from shmem global array and resolve
+ * them one by one.
+ */
+void
+FdwXactResolveAllDanglingTransactions(Oid dbid)
+{
+	List		*dangling_fdwxacts = NIL;
+	ListCell	*cell;
+	bool		n_resolved = 0;
+	int			i;
+
+	Assert(OidIsValid(dbid));
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/*
+	 * Walk over the global array to make the list of dangling transactions
+	 * of which corresponding local transaction is on the given database.
+	 */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fxact = FdwXactCtl->fdw_xacts[i];
+
+		/*
+		 * Append the fdwxact entry on the given database to the list if
+		 * it's handled by nobody and the corresponding local transaction
+		 * is not part of the prepared transaction.
+		 */
+		if (fxact->dbid == dbid &&
+			fxact->held_by == InvalidBackendId &&
+			!TwoPhaseExists(fxact->local_xid))
+			dangling_fdwxacts = lappend(dangling_fdwxacts, fxact);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/* Return if there is no foreign transaction we need to resolve */
+	if (dangling_fdwxacts == NIL)
+		return;
+
+	foreach(cell, dangling_fdwxacts)
+	{
+		FdwXact fdwxact = (FdwXact) lfirst(cell);
+		FdwXactState *state;
+
+		state = create_fdw_xact_state();
+		state->serverid = fdwxact->serverid;
+		state->userid = fdwxact->userid;
+		state->umid = fdwxact->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		FdwXactResolveForeignTransaction(state, fdwxact, ERROR);
+
+		n_resolved++;
+	}
+
+	list_free(dangling_fdwxacts);
+
+	elog(DEBUG2, "resolved %d dangling foreign xacts", n_resolved);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ *
+ * In commit case, we have already prepared transactions on the foreign
+ * servers during pre-commit. And that prepared transactions will be
+ * resolved by the resolver process. So we don't do anything about the
+ * foreign transaction.
+ *
+ * In abort case, user requested rollback or we changed over rollback
+ * due to error during commit. To close current foreign transaction anyway
+ * we call rollback API to every foreign transaction. If we raised an error
+ * during preparing and came to here, it's possible that some entries of
+ * FdwXactParticipants already registered its FdwXact entry. If there is
+ * we leave them as dangling transaction and ask the resolver process to
+ * process them.
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		int left_fdwxacts = 0;
+		FdwXactState *state = create_fdw_xact_state();
+
+		foreach (lcell, FdwXactAtomicCommitParticipants)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * Count FdwXact entries that we registered to shared memory array
+			 * in this transaction.
+			 */
+			if (fdw_part->fdw_xact)
+			{
+				/*
+				 * The status of foreign transaction must be either preparing
+				 * or prepared. In any case, since we have registered FdwXact
+				 * entry we leave them to the resolver process. For the preparing
+				 * state, since the foreign transaction might not close yet we
+				 * fall through and call rollback API. For the prepared state,
+				 * since the foreign transaction has closed we don't need to do
+				 * anything.
+				 */
+				Assert(fdw_part->fdw_xact->status == FDW_XACT_PREPARING ||
+					   fdw_part->fdw_xact->status == FDW_XACT_PREPARED);
+
+				left_fdwxacts++;
+				if (fdw_part->fdw_xact->status == FDW_XACT_PREPARED)
+					continue;
+			}
+
+			state->serverid = fdw_part->server->serverid;
+			state->userid = fdw_part->usermapping->userid;
+			state->umid = fdw_part->usermapping->umid;
+			state->fdw_state = fdw_part->fdw_state;
+
+			/*
+			 * Rollback all current foreign transaction. Since we're rollbacking
+			 * the transaction it's too late even if we raise an error here.
+			 * So we log it as warning.
+			 */
+			if (!fdw_part->rollback_foreign_xact(state))
+				ereport(WARNING,
+						(errmsg("could not abort transaction on server \"%s\"",
+								fdw_part->server->servername)));
+		}
+
+		/* If we left some FdwXact entries, ask the resolver process */
+		if (left_fdwxacts > 0)
+		{
+			ereport(WARNING,
+					(errmsg("might have left %u foreign transactions in in-doubt status",
+							left_fdwxacts)));
+			fdwxact_maybe_launch_resolver(true);
+		}
+	}
+
+	ForgetAllFdwXactParticipants();
+	FdwXactAtomicCommitReady = false;
+}
+
+/*
+ * AtPrepare_FdwXacts
+ *
+ * If there are foreign servers involved in the transaction, this function
+ * prepares transactions on those servers.
+ *
+ * Note that it can happen that the transaction aborts after we prepared part
+ * of participants. In this case since we can change to abort we cannot forget
+ * FdwXactAtomicCommitParticipants here. These are processed by the resolver process
+ * during aborting, or at EOXact_FdwXacts.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (!IsAtomicCommitEnabled())
+		return;
+
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (FdwXactAtomicCommitRequired() &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_T_R_INTEGRITY_CONSTRAINT_VIOLATION),
+				 errmsg("can not prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * FdwXactResolveForeignTransaction
+ *
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * handler routine. The foreign transaction can be a dangling transaction
+ * that is not interested by nobody. If the fate of foreign transaction is
+ * not determined yet, it'sdetermined according to the status of corresponding
+ * local transaction.
+ *
+ * If the resolution is successful, remove the foreign transaction entry from
+ * the shared memory and also remove the corresponding on-disk file.
+ */
+static bool
+FdwXactResolveForeignTransaction(FdwXactState *state, FdwXact fdwxact,
+								 int elevel)
+{
+	ForeignServer		*server;
+	ForeignDataWrapper	*fdw;
+	FdwRoutine			*fdw_routine;
+	bool		is_commit;
+	bool		ret;
+
+	Assert(fdwxact);
+
+	/*
+	 * Determine whether we commit or abort this foreign transaction.
+	 */
+	if (fdwxact->status == FDW_XACT_COMMITTING_PREPARED)
+		is_commit = true;
+	else if (fdwxact->status == FDW_XACT_ABORTING_PREPARED)
+		is_commit = false;
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	else if (TransactionIdDidCommit(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_COMMITTING_PREPARED;
+		is_commit = true;
+	}
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_ABORTING_PREPARED;
+		is_commit = false;
+	}
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+		is_commit = false;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction
+	 * state is neither committing or aborting. This should not
+	 * happen because we cannot determine to do commit or abort for
+	 * foreign transaction associated with the in-progress local
+	 * transaction.
+	 */
+	else
+		ereport(ERROR,
+				(errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid)));
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	/* Resolve the foreign transaction */
+	Assert(fdw_routine->ResolveForeignTransaction);
+
+	ret = fdw_routine->ResolveForeignTransaction(state, is_commit);
+
+	if (!ret)
+	{
+		ereport(elevel,
+				(errmsg("could not %s a prepared foreign transaction on server \"%s\"",
+						is_commit ? "commit" : "rollback", server->servername),
+				 errdetail("local transaction id is %u, connected by user id %u",
+						   fdwxact->local_xid, fdwxact->userid)));
+	}
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdw_xact(fdwxact);
+	LWLockRelease(FdwXactLock);
+
+	return ret;
+}
+
+static FdwXactState *
+create_fdw_xact_state(void)
+{
+	FdwXactState *state;
+
+	state = palloc(sizeof(FdwXactState));
+	state->serverid = InvalidOid;
+	state->userid = InvalidOid;
+	state->umid = InvalidOid;
+	state->fdwxact_id = NULL;
+	state->fdw_state = NULL;
+
+	return state;
+}
+
+/*
+ * Return one FdwXact entry that matches to given arguments, otherwise
+ * return NULL. Since this function search FdwXact entry by unique key
+ * all arguments should be valid.
+ */
+static FdwXact
+get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				 bool need_lock)
+{
+	List	*fdw_xact_list;
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, need_lock);
+
+	/* Could not find entry */
+	if (fdw_xact_list == NIL)
+		return NULL;
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdw_xact_list) == 1);
+
+	return (FdwXact) linitial(fdw_xact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdw_xact_exists(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdw_xact_list;
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, true);
+
+	return fdw_xact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_prepared_fdw_xacts.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdw_xacts(int *length)
+{
+	List		*all_fdw_xacts;
+	ListCell	*lc;
+	FdwXact		fdw_xacts;
+	int			num_fdw_xacts = 0;
+
+	Assert(length != NULL);
+
+	/* Get all entries */
+	all_fdw_xacts = get_fdw_xacts(InvalidOid, InvalidTransactionId,
+								  InvalidOid, InvalidOid, true);
+
+	if (all_fdw_xacts == NIL)
+	{
+		*length = 0;
+		return NULL;
+	}
+
+	fdw_xacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdw_xacts));
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdw_xacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdw_xacts + num_fdw_xacts, fx,
+			   sizeof(FdwXactData));
+		num_fdw_xacts++;
+	}
+
+	*length = num_fdw_xacts;
+	list_free(all_fdw_xacts);
+
+	return fdw_xacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return
+ * NIL.
+ */
+static List*
+get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			  bool need_lock)
+{
+	int i;
+	List	*fdw_xact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact	fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool	matches = true;
+
+		/* xid */
+		if (xid != InvalidTransactionId && xid != fdw_xact->local_xid)
+			matches = false;
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdw_xact->dbid != dbid)
+			matches = false;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdw_xact->serverid)
+			matches = false;
+
+		/* userid */
+		if (OidIsValid(userid) && fdw_xact->userid != userid)
+			matches = false;
+
+		/* Append it if matched */
+		if (matches)
+			fdw_xact_list = lappend(fdw_xact_list, fdw_xact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdw_xact_list;
+}
+
+/*
+ * fdw_xact_redo
+ * Apply the redo log for a foreign transaction.
+ */
+void
+fdw_xact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDW_XACT_REMOVE)
+	{
+		xl_fdw_xact_remove *record = (xl_fdw_xact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier with in the form
+ * of "fx_<random number>_<xid>_<serverid>_<userid> whose length is always
+ * less than NAMEDATALEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+generate_fdw_xact_identifier(TransactionId xid, Oid serverid, Oid userid)
+{
+	char*	fdw_xact_id;
+
+	fdw_xact_id = (char *)palloc0(FDW_XACT_ID_MAX_LEN * sizeof(char));
+
+	snprintf(fdw_xact_id, FDW_XACT_ID_MAX_LEN, "%s_%ld_%u_%d_%d",
+			 "fx", Abs(random()), xid, serverid, userid);
+	fdw_xact_id[strlen(fdw_xact_id)] = '\0';
+
+	return fdw_xact_id;
+}
+
+/*
+ * CheckPointFdwXact
+ *
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * In order to avoid disk I/O while holding a light weight lock, the function
+ * first collects the files which need to be synced under FdwXactLock and then
+ * syncs them after releasing the lock. This approach creates a race condition:
+ * after releasing the lock, and before syncing a file, the corresponding
+ * foreign transaction entry and hence the file might get removed. The function
+ * checks whether that's true and ignores the error if so.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdw_xacts = 0;
+
+	/* Quick get-away, before taking lock */
+	if (max_prepared_foreign_xacts <= 0)
+		return;
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/* Another quick, before we allocate memory */
+	if (FdwXactCtl->numFdwXacts <= 0)
+	{
+		LWLockRelease(FdwXactLock);
+		return;
+	}
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		FdwXact		fxact = FdwXactCtl->fdw_xacts[cnt];
+
+		if ((fxact->valid || fxact->inredo) &&
+			!fxact->ondisk &&
+			fxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fxact->dbid, fxact->local_xid,
+								fxact->serverid, fxact->userid,
+								buf, len);
+			fxact->ondisk = true;
+			fxact->insert_start_lsn = InvalidXLogRecPtr;
+			fxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdw_xacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDW_XACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdw_xacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdw_xacts,
+							 serialized_fdw_xacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDW_XACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDW_XACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * ProcessFdwXactBuffer
+ *
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid = ShmemVariableCache->nextXid;
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(insert_start_lsn != InvalidXLogRecPtr);
+
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid, true);
+		if (buf == NULL)
+		{
+			ereport(WARNING,
+					(errmsg("removing corrupt fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+			return NULL;
+		}
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return thecontents in
+ * a structure allocated in-memory. Otherwise return NULL. The structure can
+ * be later freed by the caller.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				bool give_warnings)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+	{
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not stat FDW transaction state file \"%s\": %m",
+							path)));
+		return NULL;
+	}
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdw_xact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+	{
+		CloseTransientFile(fd);
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+		return NULL;
+	}
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+	{
+		CloseTransientFile(fd);
+		return NULL;
+	}
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_READ);
+	if (read(fd, buf, stat.st_size) != stat.st_size)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not read FDW transaction state file \"%s\": %m",
+					  path)));
+		return NULL;
+	}
+
+	pgstat_report_wait_end();
+	CloseTransientFile(fd);
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+	{
+		pfree(buf);
+		return NULL;
+	}
+
+	/* Check if the contents is an expected data */
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fxact_file_data->dbid  != dbid ||
+		fxact_file_data->serverid != serverid ||
+		fxact_file_data->userid != userid ||
+		fxact_file_data->local_xid != xid)
+	{
+		ereport(WARNING,
+			(errmsg("invalid foreign transaction state file \"%s\"",
+					path)));
+		CloseTransientFile(fd);
+		pfree(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+/*
+ * PrescanFdwXacts
+ *
+ * Scan the all foreign transactions directory for oldest active transaction.
+ * This is run during database startup, after we completed reading WAL.
+ * ShmemVariableCache->nextXid has been set to one more than the highest XID
+ * for which evidence exists in WAL.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	TransactionId nextXid = ShmemVariableCache->nextXid;
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+		 strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			TransactionId local_xid;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/*
+			 * Remove a foreign prepared transaction file corresponding to an
+			 * XID, which is too new.
+			 */
+			if (TransactionIdFollowsOrEquals(local_xid, nextXid))
+			{
+				ereport(WARNING,
+						(errmsg("removing future foreign prepared transaction file \"%s\"",
+								clde->d_name)));
+				RemoveFdwXactFile(dbid, local_xid, serverid, userid, true);
+				continue;
+			}
+
+			if (TransactionIdPrecedesOrEquals(local_xid, oldestActiveXid))
+				oldestActiveXid = local_xid;
+		}
+	}
+
+	FreeDir(cldir);
+	return oldestActiveXid;
+}
+
+/*
+ * restoreFdwXactData
+ *
+ * Scan pg_fdw_xact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid, bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * FdwXactRedoAdd
+ *
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fxact = insert_fdw_xact(fxact_data->dbid, fxact_data->local_xid,
+							fxact_data->serverid, fxact_data->userid,
+							fxact_data->umid, fxact_data->fdw_xact_id);
+
+	/*
+	 * Set status as preparing, since we do not know the xact status
+	 * right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->insert_start_lsn = start_lsn;
+	fxact->insert_end_lsn = end_lsn;
+	fxact->inredo = true;	/* added in redo */
+	fxact->valid = false;
+	fxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * FdwXactRedoRemove
+ *
+ * Remove the corresponding fdw_xact entry from FdwXactCtl.
+ * Also remove fdw_xact file if a foreign transaction was saved
+ * via an earlier checkpoint.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdw_xact(dbid, xid, serverid, userid,
+							   false);
+
+	if (fdwxact == NULL)
+		return;
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdw_xact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(int *newval, void **extra, GucSource source)
+{
+	ForeignTwophaseCommitLevel newForeignTwophaseCommitLevel = *newval;
+
+		/* Parameter check */
+	if (newForeignTwophaseCommitLevel > FOREIGN_TWOPHASE_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdw_xacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_prepared_fdw_xacts(PG_FUNCTION_ARGS)
+{
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdw_xacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdw_xacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(6, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdw_xacts = get_all_fdw_xacts(&num_fdw_xacts);
+		status->num_xacts = num_fdw_xacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdw_xact = &status->fdw_xacts[status->cur_xact++];
+		Datum		values[6];
+		bool		nulls[6];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdw_xact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdw_xact->dbid);
+		values[1] = TransactionIdGetDatum(fdw_xact->local_xid);
+		values[2] = ObjectIdGetDatum(fdw_xact->serverid);
+		values[3] = ObjectIdGetDatum(fdw_xact->userid);
+		switch (fdw_xact->status)
+		{
+			case FDW_XACT_PREPARING:
+				xact_status = "prepared";
+				break;
+			case FDW_XACT_COMMITTING_PREPARED:
+				xact_status = "committing";
+				break;
+			case FDW_XACT_ABORTING_PREPARED:
+				xact_status = "aborting";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		/* should this be really interpreted by FDW */
+		values[5] = PointerGetDatum(cstring_to_text_with_len(fdw_xact->fdw_xact_id,
+															 strlen(fdw_xact->fdw_xact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXactState *state;
+	UserMapping		*usermapping;
+	FdwXact			fdwxact;
+	bool			ret;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, true);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	usermapping = GetUserMapping(userid, serverid);
+
+	state = create_fdw_xact_state();
+	state->serverid = serverid;
+	state->userid = userid;
+	state->umid = usermapping->umid;
+
+	ret = FdwXactResolveForeignTransaction(state, fdwxact, LOG);
+
+	PG_RETURN_BOOL(ret);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, false);
+	if (fdwxact == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("could not find foreign transaction entry"))));
+
+	remove_fdw_xact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_VOID();
+}
diff --git a/src/backend/access/fdwxact/fdwxact_launcher.c b/src/backend/access/fdwxact/fdwxact_launcher.c
new file mode 100644
index 0000000..39f351b
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact_launcher.c
@@ -0,0 +1,641 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * There is a shared memory area where the information of resolver process
+ * is stored. Requesting of starting new resolver process by backend process
+ * is done via that shared memory area. Note that the launcher is assuming
+ * that there is no more than one starting request for a database.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/fdwxact_launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launcher_sigusr2(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid, int slot);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+Datum pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS);
+
+/*
+ * Wake up the launcher process to retry launch. This is used by
+ * the resolver process is being stopped.
+ */
+void
+FdwXactLauncherWakeupToRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request resolution. This is
+ * used by the backend process.
+ */
+void
+FdwXactLauncherWakeupToRequest(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->FdwXactActiveQueue));
+		SHMQueueInit(&(FdwXactRslvCtl->FdwXactRetryQueue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR1: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a foreign_xact_resolution_retry_interval
+		 * but always try to start by the backend request.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool launched;
+
+			ResetLatch(MyLatch);
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher launch",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver worker
+ * if not running yet. A foreign transaction resolver worker is responsible
+ * for resolution of foreign transaction that are registered on a database.
+ * So if a resolver worker already is launched, we don't need to launch new
+ * one.
+ */
+void
+fdwxact_maybe_launch_resolver(bool ignore_error)
+{
+	FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->pid != InvalidPid &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/*
+	 * If we found the resolver for my database, we don't need to launch new
+	 * one but wake running worker up.
+	 */
+	if (found)
+	{
+		SetLatch(resolver->latch);
+
+		elog(DEBUG1, "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		return;
+	}
+
+	/* Looking for unused resolver slot */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	/*
+	 * However if there are no more free worker slots, inform user about it before
+	 * exiting.
+	 */
+	if (!found)
+	{
+		LWLockRelease(FdwXactResolverLock);
+
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+		return;
+	}
+
+	Assert(resolver->pid == InvalidPid);
+
+	/* Found a new resolver process */
+	resolver->dbid = MyDatabaseId;
+	resolver->in_use = true;
+
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Wake up launcher */
+	FdwXactLauncherWakeupToRequest();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid' at 'slot' if given. If slot is negative value we find an unused slot.
+ * Note that caller must hold FdwXactResolverLock in exclusive mode.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid, int slot)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int launch_slot = slot;
+
+	/* If slot number is invalid, we find an unused slot */
+	if (launch_slot < 0)
+	{
+		int i;
+
+		for (i = 0; i < max_foreign_xact_resolvers; i++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+			if (resolver->in_use && resolver->dbid == dbid)
+				return;
+
+			if (!resolver->in_use)
+			{
+				launch_slot = i;
+				break;
+			}
+		}
+	}
+
+	/* No unused found */
+	if (launch_slot < 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[launch_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_main_arg = Int32GetDatum(launch_slot);
+	bgw.bgw_notify_pid = (Datum) 0;
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch all foreign transaction resolvers that are required by backend process
+ * but not running. Return true if we launch any resolver.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	int i, j;
+	int num_launches = 0;
+	int num_unused_slots = 0;
+	int num_dbs = 0;
+	bool launched = false;
+	Oid	*dbs_to_launch;
+	Oid	*dbs_having_worker = palloc0(sizeof(Oid) * max_foreign_xact_resolvers);
+
+	/*
+	 * Launch resolver workers on the databases that are requested
+	 * by backend processes while looking unused slots.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* Remember unused worker slots */
+		if (!resolver->in_use)
+		{
+			num_unused_slots++;
+			continue;
+		}
+
+		/* Remember databases that are having a resolve worker, fall through */
+		if (OidIsValid(resolver->dbid))
+			dbs_having_worker[num_dbs++] = resolver->dbid;
+
+		/* Launch the backend-requested worker */
+		if (resolver->in_use &&
+			OidIsValid(resolver->dbid) &&
+			resolver->pid == InvalidPid)
+		{
+			fdwxact_launch_resolver(resolver->dbid, i);
+			launched = true;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* quick exit if no unused slot */
+	if (num_unused_slots == 0)
+		return launched;
+
+	/*
+	 * Launch the stopped resolver on the database that has unresolved
+	 * foreign transaction but doesn't have any resolver. Scanning
+	 * all FdwXact entries could take time but it's harmless for the
+	 * relaunch case.
+	 */
+	dbs_to_launch = (Oid *) palloc(sizeof(Oid) * num_unused_slots);
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool found = false;
+
+		/* unused slot is full */
+		if (num_launches > num_unused_slots)
+			break;
+
+		for (j = 0; j < num_dbs; j++)
+		{
+			if (dbs_having_worker[j] == fdw_xact->dbid)
+			{
+				found = true;
+				break;
+			}
+		}
+
+		/* Register the database if any resolvers aren't working on that */
+		if (!found)
+			dbs_to_launch[num_launches++] = fdw_xact->dbid;
+	}
+
+	/* Launch resolver process for a database at any worker slot */
+	for (i = 0; i < num_launches; i++)
+	{
+		fdwxact_launch_resolver(dbs_to_launch[i], -1);
+		launched = true;
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+
+/*
+ * Returns activity of foreign transaction resolvers, including pids, the number
+ * of tasks and the last resolution time.
+ */
+Datum
+pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/fdwxact_resolver.c b/src/backend/access/fdwxact/fdwxact_resolver.c
new file mode 100644
index 0000000..0b754da
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact_resolver.c
@@ -0,0 +1,331 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for every databases.
+ *
+ * A resolver process continues to resolve foreign transactions on a database
+ * It resolves two types of foreign transactions: on-line foreign transaction
+ * and dangling foreign transaction. The on-line foreign transaction is a
+ * foreign transaction that a concurrent backend process is waiting for
+ * resolution. The dangling transaction is a foreign transaction that corresponding
+ * distributed transaction ended up in in-doubt state. A resolver process
+ * doesn' exit as long as there is at least one unresolved foreign transaction
+ * on the database even if the timeout has come.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/fdwxact_resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+
+//static MemoryContext ResolveContext = NULL;
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FdwXactRslvLoop(void);
+static long FdwXactRslvComputeSleepTime(TimestampTz now);
+static void FdwXactRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+	FdwXactLauncherWakeupToRetry();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	TIMESTAMP_NOBEGIN(MyFdwXactResolver->last_resolved_time);
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FdwXactRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FdwXactRslvLoop(void)
+{
+	TimestampTz last_retry_time = 0;
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time;
+		bool		resolved;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		/* Resolve one distributed transaction */
+		StartTransactionCommand();
+		resolved = FdwXactResolveDistributedTransaction(MyDatabaseId, true);
+		CommitTransactionCommand();
+
+		now = GetCurrentTimestamp();
+
+		/* Update my state */
+		if (resolved)
+			MyFdwXactResolver->last_resolved_time = now;
+
+		if (TimestampDifferenceExceeds(last_retry_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			StartTransactionCommand();
+			resolved = FdwXactResolveDistributedTransaction(MyDatabaseId, false);
+			CommitTransactionCommand();
+
+			last_retry_time = GetCurrentTimestamp();
+
+			/* Update my state */
+			if (resolved)
+				MyFdwXactResolver->last_resolved_time = last_retry_time;
+		}
+
+		/* Check for fdwxact resolver timeout */
+		FdwXactRslvCheckTimeout(now);
+
+		/*
+		 * If we have resolved any distributed transaction we go the next
+		 * without both resolving dangling transaction and sleeping because
+		 * there might be other on-line transactions waiting to be resolved.
+		 */
+		if (!resolved)
+		{
+			/* Resolve dangling transactions as mush as possible */
+			StartTransactionCommand();
+			FdwXactResolveAllDanglingTransactions(MyDatabaseId);
+			CommitTransactionCommand();
+
+			sleep_time = FdwXactRslvComputeSleepTime(now);
+
+			MemoryContextResetAndDeleteChildren(resolver_ctx);
+			MemoryContextSwitchTo(TopMemoryContext);
+
+			rc = WaitLatch(MyLatch,
+						   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+						   sleep_time,
+						   WAIT_EVENT_FDW_XACT_RESOLVER_MAIN);
+
+			if (rc & WL_POSTMASTER_DEATH)
+				proc_exit(1);
+		}
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FdwXactRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	/*
+	 * Reached to the timeout. We exit if there is no more both pending on-line
+	 * transactions and dangling transactions.
+	 */
+	if (!fdw_xact_exists(InvalidTransactionId, MyDatabaseId, InvalidOid,
+						 InvalidOid))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyFdwXactResolver->dbid))));
+		CommitTransactionCommand();
+
+		fdwxact_resolver_detach();
+		proc_exit(0);
+	}
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. Return the sleep time
+ * in milliseconds, -1 means that we reached to the timeout and should exits
+ */
+static long
+FdwXactRslvComputeSleepTime(TimestampTz now)
+{
+	static TimestampTz	wakeuptime = 0;
+	long	sleeptime;
+	long	sec_to_timeout;
+	int		microsec_to_timeout;
+
+	if (now >= wakeuptime)
+		wakeuptime = TimestampTzPlusMilliseconds(now,
+												 foreign_xact_resolution_retry_interval);
+
+	/* Compute relative time until wakeup. */
+	TimestampDifference(now, wakeuptime,
+						&sec_to_timeout, &microsec_to_timeout);
+
+	sleeptime = sec_to_timeout * 1000 + microsec_to_timeout / 1000;
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index c2db19b..fb63471 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2629,10 +2629,6 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 		heap_freetuple(heaptup);
 	}
 
-	/* Make note that we've wrote on non-temprary relation */
-	if (RelationNeedsWAL(relation))
-		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
-
 	return HeapTupleGetOid(tup);
 }
 
@@ -3457,10 +3453,6 @@ l1:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
-	/* Make note that we've wrote on non-temprary relation */
-	if (RelationNeedsWAL(relation))
-		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
-
 	return HeapTupleMayBeUpdated;
 }
 
@@ -4411,10 +4403,6 @@ l2:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
-	/* Make note that we've wrote on non-temprary relation */
-	if (RelationNeedsWAL(relation))
-		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
-
 	bms_free(hot_attrs);
 	bms_free(proj_idx_attrs);
 	bms_free(key_attrs);
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index 5514db1..742e825 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -8,9 +8,9 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
-	   gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
-	   mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
-	   smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdwxactdesc.o \
+	genericdesc.o  gindesc.o gistdesc.o hashdesc.o heapdesc.o \
+	logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \
+	seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000..7061bba
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,65 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdw_xactdesc.c
+ *		PostgreSQL distributed transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdw_xactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdw_xact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		FdwXactOnDiskData *fdw_insert_xlog = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_insert_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_insert_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_insert_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_insert_xlog->local_xid);
+		/* TODO: This should be really interpreted by each FDW */
+
+		/*
+		 * TODO: we also need to assess whether we want to add this
+		 * information
+		 */
+		appendStringInfo(buf, " foreign transaction info: %s",
+						 fdw_insert_xlog->fdw_xact_id);
+	}
+	else
+	{
+		xl_fdw_xact_remove *fdw_remove_xlog = (xl_fdw_xact_remove *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_remove_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_remove_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_remove_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_remove_xlog->xid);
+	}
+
+}
+
+const char *
+fdw_xact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDW_XACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDW_XACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 00741c7..4a9ab3d 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -112,14 +112,16 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_prepared_xacts=%d max_locks_per_xact=%d "
 						 "wal_level=%s wal_log_hints=%s "
-						 "track_commit_timestamp=%s",
+						 "track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_prepared_xacts,
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47..f15c83a 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -12,9 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
-	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
-	xact.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o generic_xlog.o multixact.o \
+	parallel.o rmgr.o slru.o subtrans.o timeline.o transam.o twophase.o \
+	twophase_rmgr.o varsup.o xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9368b56..8b360b1 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -9,6 +9,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
 #include "access/generic_xlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 3942734..bc4e109 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -844,6 +845,35 @@ TwoPhaseGetGXact(TransactionId xid)
 }
 
 /*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
+/*
  * TwoPhaseGetDummyProc
  *		Get the dummy backend ID for prepared transaction specified by XID
  *
@@ -2316,6 +2346,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2375,6 +2411,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 8c1621d..9dca0f5 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1131,6 +1132,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1139,6 +1141,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsAtomicCommitReady();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1177,12 +1180,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1340,6 +1344,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -1990,6 +2002,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2146,6 +2161,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2233,6 +2249,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2422,6 +2440,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2627,6 +2646,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7375a78..9de3bcc 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
 #include "access/subtrans.h"
@@ -5267,6 +5268,7 @@ BootStrapXLOG(void)
 	ControlFile->MaxConnections = MaxConnections;
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6354,6 +6356,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -6878,14 +6883,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdw_xact, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -7077,7 +7083,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7583,6 +7592,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7901,6 +7911,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9217,6 +9230,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9650,7 +9664,8 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9682,6 +9697,7 @@ XLogReportParameters(void)
 		ControlFile->MaxConnections = MaxConnections;
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9887,6 +9903,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -10085,6 +10102,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->MaxConnections = xlrec.MaxConnections;
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index a03b005..47e9317 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -291,6 +291,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_prepared_fdw_xacts AS
+       SELECT * FROM pg_prepared_fdw_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
 	l.objoid, l.classoid, l.objsubid,
@@ -773,6 +776,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_fdwxact_resolvers AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_fdwxact_resolver() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index e5dd995..dac1e3a 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
@@ -1093,6 +1094,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1407,6 +1420,16 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
 	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srv->serverid,
+						useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
+	/*
 	 * Do the deletion
 	 */
 	object.classId = UserMappingRelationId;
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index 0bcb237..058bc0a 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "catalog/partition.h"
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_type.h"
@@ -749,7 +750,10 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+		FdwXactMarkForeignServerAccessed(partRelInfo->ri_RelationDesc, 0, true);
+	}
 
 	MemoryContextSwitchTo(oldContext);
 
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 2ec7fcb..4578bc0 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,10 +226,16 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+
+	}
 	else
 		fdwroutine->BeginForeignScan(scanstate, eflags);
 
+	FdwXactMarkForeignServerAccessed(scanstate->ss.ss_currentRelation,
+									 eflags, node->operation != CMD_SELECT);
+
 	return scanstate;
 }
 
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 528f587..66c3699 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "commands/trigger.h"
@@ -44,6 +45,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "storage/bufmgr.h"
@@ -485,6 +487,10 @@ ExecInsert(ModifyTableState *mtstate,
 								HEAP_INSERT_SPECULATIVE,
 								NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self),
 												   estate, true, &specConflict,
@@ -530,6 +536,10 @@ ExecInsert(ModifyTableState *mtstate,
 								estate->es_output_cid,
 								0, NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self),
@@ -722,6 +732,11 @@ ldelete:;
 							 true /* wait for commit */ ,
 							 &hufd,
 							 changingPart);
+
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case HeapTupleSelfUpdated:
@@ -1210,6 +1225,11 @@ lreplace:;
 							 estate->es_crosscheck_snapshot,
 							 true /* wait for commit */ ,
 							 &hufd, &lockmode);
+
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case HeapTupleSelfUpdated:
@@ -2321,6 +2341,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 															 fdw_private,
 															 i,
 															 eflags);
+
+			/* Mark this transaction modified data on the foreign server */
+			FdwXactMarkForeignServerAccessed(resultRelInfo->ri_RelationDesc,
+											 eflags, true);
 		}
 
 		resultRelInfo++;
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index a0bcc04..b2097ad 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -155,6 +155,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMapping - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index d2b695e..b722b9a 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -15,6 +15,8 @@
 #include <unistd.h>
 
 #include "libpq/pqsignal.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 8de603d..3f59fdd 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3484,6 +3484,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_LAUNCHER_MAIN:
 			event_name = "LogicalLauncherMain";
 			break;
@@ -3675,6 +3681,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -3890,6 +3899,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDW_XACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 688f462..883ad85 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -896,6 +898,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -971,12 +977,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index afb4972..960fd6a 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -154,6 +154,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDW_XACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 0c86a58..c5610ee 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -150,6 +152,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
 		size = add_size(size, BackendRandomShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -270,6 +274,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 	SyncScanShmemInit();
 	AsyncShmemInit();
 	BackendRandomShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 908f62d..cc578b2 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -90,6 +90,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -245,6 +247,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1323,6 +1326,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	volatile TransactionId replication_slot_xmin = InvalidTransactionId;
 	volatile TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	volatile TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1384,6 +1388,7 @@ GetOldestXmin(Relation rel, int flags)
 	/* fetch into volatile var while ProcArrayLock is held */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1434,6 +1439,15 @@ GetOldestXmin(Relation rel, int flags)
 		result = replication_slot_xmin;
 
 	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDW_XACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
+	/*
 	 * After locks have been released and defer_cleanup_age has been applied,
 	 * check whether we need to back up further to make logical decoding
 	 * possible. We need to do so if we're computing the global limit (rel =
@@ -3016,6 +3030,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index e6025ec..a42d06e 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -50,3 +50,5 @@ OldSnapshotTimeMapLock				42
 BackendRandomLock					43
 LogicalRepWorkerLock				44
 CLogTruncationLock					45
+FdwXactLock					46
+FdwXactResolverLock			47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 6f9aaa5..8e55dad 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -398,6 +399,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* initialize fields for fdw xact */
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -799,6 +804,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 6e13d14..1302d16 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -2971,6 +2973,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 2317e8b..7651352 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/transam.h"
@@ -378,6 +379,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 };
 
 /*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
+/*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
  */
@@ -659,6 +679,10 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2235,6 +2259,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4056,6 +4126,16 @@ static struct config_enum ConfigureNamesEnum[] =
 	},
 
 	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+			gettext_noop("Sets the usage of two-phase commit protocol for distributed transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
+	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
 			NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 4e61bc6..88cdc85 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -121,6 +121,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -287,6 +289,20 @@
 
 
 #------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
+#------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index ad06e8e..ca3eb62 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index ab5cb7f..609578c 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -209,6 +209,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdw_xact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 895a51f..7df88e0 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -306,6 +306,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_worker_processes);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 6fb403a..6d867c8 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -730,6 +730,7 @@ GuessControlValues(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -957,6 +958,7 @@ RewriteControlFile(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* Contents are protected with a CRC */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca..b616cea 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000..0928f4c
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,147 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL distributed transaction manager
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDW_XACT_H
+#define FDW_XACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+#define	FDW_XACT_NOT_WAITING		0
+#define	FDW_XACT_WAITING			1
+#define	FDW_XACT_WAITING_RETRY		2
+#define	FDW_XACT_WAIT_COMPLETE		3
+
+#define FdwXactEnabled() (max_prepared_foreign_xacts > 0)
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDW_XACT_ID_MAX_LEN 200
+
+/* Enum to track the status of prepared foreign transaction */
+typedef enum
+{
+	FDW_XACT_INITIAL,
+	FDW_XACT_PREPARING,					/* foreign transaction is being prepared */
+	FDW_XACT_PREPARED,					/* foreign transaction is prepared */
+	FDW_XACT_COMMITTING_PREPARED,		/* foreign prepared transaction is to
+										 * be committed */
+	FDW_XACT_ABORTING_PREPARED, /* foreign prepared transaction is to be
+								 * aborted */
+} FdwXactStatus;
+
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER,	/* use twophase commit where available */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support twophase
+								 * commit */
+} ForeignTwophaseCommitLevel;
+
+/* Shared memory entry for a prepared or being prepared foreign transaction */
+typedef struct FdwXactData *FdwXact;
+
+typedef struct FdwXactData
+{
+	FdwXact		fxact_free_next;	/* Next free FdwXact entry */
+	FdwXact		fxact_next;			/* Pointer to the neext FdwXact entry accosiated
+									 * with the same transaction */
+	Oid				dbid;			/* database oid where to find foreign server
+									 * and user mapping */
+	TransactionId	local_xid;		/* XID of local transaction */
+	Oid				serverid;		/* foreign server where transaction takes place */
+	Oid				userid;			/* user who initiated the foreign transaction */
+	Oid				umid;
+	FdwXactStatus 	status;			/* The state of the foreign transaction. This
+									 * doubles as the action to be taken on this entry. */
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;		/* XLOG offset of inserting this entry start */
+	XLogRecPtr	insert_end_lsn;		/* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN];		/* prepared transaction identifier */
+} FdwXactData;
+
+/* Shared memory layout for maintaining foreign prepared transaction entries. */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		freeFdwXacts;
+
+	/* Number of valid foreign transaction entries */
+	int			numFdwXacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdw_xacts[FLEXIBLE_ARRAY_MEMBER];		/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+typedef struct FdwXactState
+{
+	Oid		serverid;
+	Oid		userid;
+	Oid		umid;
+	char	*fdwxact_id;
+	void	*fdw_state;		/* foreign-data wrapper can keep state here */
+} FdwXactState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern bool fdw_xact_exists(TransactionId xid, Oid dboid, Oid serverid,
+				Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern void PreCommit_FdwXacts(void);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern bool FdwXactResolveDistributedTransaction(Oid dbid, bool is_active);
+extern void FdwXactResolveAllDanglingTransactions(Oid dbid);
+extern bool FdwXactIsAtomicCommitReady(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void FdwXactMarkForeignServerAccessed(Relation rel, int flags, bool modified);
+extern bool check_foreign_twophase_commit(int *newval, void **extra,
+										  GucSource source);
+
+#endif   /* FDW_XACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000..4ea65b2
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,32 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _FDWXACT_LAUNCHER_H
+#define _FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherWakeupToRequest(void);
+extern void FdwXactLauncherWakeupToRetry(void);
+
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+
+extern bool IsFdwXactLauncher(void);
+
+extern void fdwxact_maybe_launch_resolver(bool ignore_error);
+
+
+#endif	/* _FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000..6b2a24f
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int foreign_xact_resolver_timeout;
+
+#endif		/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000..e92b5a1
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,52 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDW_XACT_INSERT	0x00
+#define XLOG_FDW_XACT_REMOVE	0x10
+
+/* Same as GIDSIZE */
+#define FDW_XACT_MAX_ID_LEN 200
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdw_xact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+} xl_fdw_xact_remove;
+
+extern void fdw_xact_redo(XLogReaderState *record);
+extern void fdw_xact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdw_xact_identify(uint8 info);
+
+#endif	/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000..36391d4
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,67 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/resovler_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _RESOLVER_INTERNAL_H
+#define _RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t	pid;	/* this resolver's PID, or 0 if not active */
+	Oid		dbid;	/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool	in_use;
+
+	/* Stats */
+	TimestampTz	last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t	mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	*latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/*
+	 * Foreign transaction resolution queues. Protected by FdwXactLock.
+	 */
+	SHM_QUEUE	FdwXactActiveQueue;
+	SHM_QUEUE	FdwXactRetryQueue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch		*launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif	/* _RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 0bbe9879..c15dff7 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDW_XACT_ID, "Foreign Transactions", fdw_xact_redo, fdw_xact_desc, fdw_xact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 0e932da..b199c88 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 				TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 2c1b2d8..63c833d 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -105,6 +105,13 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE				(1U << 3)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 30610b3..795e85a 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -227,6 +227,7 @@ typedef struct xl_parameter_change
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index 773d9e6..3d5333a 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -178,6 +178,7 @@ typedef struct ControlFileData
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index cff58ed..d39ca1e 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5032,6 +5032,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '6053', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_fdwxact_resolver', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_fdwxact_resolver' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5737,6 +5744,22 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '6050', descr => 'view foreign transactions',
+  proname => 'pg_prepared_fdw_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,text}',
+  proargmodes => '{o,o,o,o,o,o}',
+  proargnames => '{dbid,transaction,serverid,userid,status,identifier}',
+  prosrc => 'pg_prepared_fdw_xacts' },
+{ oid => '6051', descr => 'remove foreign transaction',
+  proname => 'pg_remove_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_remove_fdw_xact' },
+{ oid => '6052', descr => 'resolve foreign transaction',
+  proname => 'pg_resolve_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_resolve_fdw_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index c14eb54..92d47bb 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/relation.h"
@@ -168,6 +169,14 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef bool (*PrepareForeignTransaction_function) (FdwXactState *state);
+typedef bool (*CommitForeignTransaction_function) (FdwXactState *state);
+typedef bool (*RollbackForeignTransaction_function) (FdwXactState *state);
+typedef bool (*ResolveForeignTransaction_function) (FdwXactState *state,
+													bool is_commit);
+typedef bool (*IsTwoPhaseCommitEnabled_function) (Oid serverid);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -235,6 +244,14 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for distributed transactions */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	ResolveForeignTransaction_function ResolveForeignTransaction;
+	IsTwoPhaseCommitEnabled_function IsTwoPhaseCommitEnabled;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
@@ -247,7 +264,6 @@ typedef struct FdwRoutine
 	ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild;
 } FdwRoutine;
 
-
 /* Functions in foreign/foreign.c */
 extern FdwRoutine *GetFdwRoutine(Oid fdwhandler);
 extern Oid	GetForeignServerIdByRelId(Oid relid);
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 3ca12e6..d030368 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -68,10 +68,10 @@ typedef struct ForeignTable
 	List	   *options;		/* ftoptions as DefElem list */
 } ForeignTable;
 
-
 extern ForeignServer *GetForeignServer(Oid serverid);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperByName(const char *name,
 							bool missing_ok);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index d59c24a..f74d1be 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -759,6 +759,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDW_XACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -832,7 +834,8 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDW_XACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -912,6 +915,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_READ,
+	WAIT_EVENT_FDW_XACT_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index cb613c8..45880b2 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -153,6 +153,16 @@ struct PGPROC
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction
+								 * resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+
+	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
 	 * their lock.
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index 75bab29..25d6a2f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDW_XACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -124,4 +126,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 								TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 668d9ef..81560bd 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -94,6 +94,8 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 735dd37..fdd6ded 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1413,6 +1413,13 @@ pg_policies| SELECT n.nspname AS schemaname,
    FROM ((pg_policy pol
      JOIN pg_class c ON ((c.oid = pol.polrelid)))
      LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace)));
+pg_prepared_fdw_xacts| SELECT f.dbid,
+    f.transaction,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.identifier
+   FROM pg_prepared_fdw_xacts() f(dbid, transaction, serverid, userid, status, identifier);
 pg_prepared_statements| SELECT p.name,
     p.statement,
     p.prepare_time,
@@ -1821,6 +1828,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_fdwxact_resolvers| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_fdwxact_resolver() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_progress_vacuum| SELECT s.pid,
     s.datid,
     d.datname,
-- 
2.10.5

From 683b47de51f23bb899c92bac8bc3947d2d262c5a Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Thu, 8 Feb 2018 11:26:46 +0900
Subject: [PATCH v21 1/4] Keep track of writing on non-temporary relation.

---
 src/backend/access/heap/heapam.c | 12 ++++++++++++
 src/include/access/xact.h        |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index fb63471..c2db19b 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2629,6 +2629,10 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 		heap_freetuple(heaptup);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleGetOid(tup);
 }
 
@@ -3453,6 +3457,10 @@ l1:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleMayBeUpdated;
 }
 
@@ -4403,6 +4411,10 @@ l2:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	bms_free(hot_attrs);
 	bms_free(proj_idx_attrs);
 	bms_free(key_attrs);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 689c57c..2c1b2d8 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -98,6 +98,11 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
-- 
2.10.5

From d3b2da59daf037fe5db43f268ce2cd582c84e390 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:48:08 +0900
Subject: [PATCH v21 4/4] Add regression tests for atomic commit.

---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index daf79a0..71c8b9d 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000..a23f120
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port', two_phase_commit 'on');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port', two_phase_commit 'on');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_prepared_fdw_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 3248603..c1cd8ae 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2288,9 +2288,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2305,7 +2308,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.10.5

From 972cf3a26d6b970d7539e4309f11c4f96b5d89fd Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:46:01 +0900
Subject: [PATCH v21 3/4] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/connection.c              | 673 ++++++++++++++++---------
 contrib/postgres_fdw/expected/postgres_fdw.out | 387 +++++++++++++-
 contrib/postgres_fdw/option.c                  |   5 +-
 contrib/postgres_fdw/postgres_fdw.c            |  60 ++-
 contrib/postgres_fdw/postgres_fdw.h            |  11 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql      | 151 +++++-
 doc/src/sgml/postgres-fdw.sgml                 |  37 ++
 7 files changed, 1069 insertions(+), 255 deletions(-)

diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index fe4893a..3264300 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -14,9 +14,12 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
-#include "catalog/pg_user_mapping.h"
 #include "access/xact.h"
+#include "catalog/pg_user_mapping.h"
+#include "commands/defrem.h"
+#include "foreign/fdwapi.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -45,7 +48,7 @@
  */
 typedef Oid ConnCacheKey;
 
-typedef struct ConnCacheEntry
+struct ConnCacheEntry
 {
 	ConnCacheKey key;			/* hash key (must be first) */
 	PGconn	   *conn;			/* connection to foreign server, or NULL */
@@ -56,9 +59,19 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		am_participant_of_ac;	/* true if fdwxact code control the transaction */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
-} ConnCacheEntry;
+};
+
+typedef struct PgFdwXactState
+{
+	Oid		serverid;
+	Oid		userid;
+	Oid		umid;
+	ConnCacheEntry	*conn;
+} PgFdwXactState;
 
 /*
  * Connection cache (initialized on first use)
@@ -69,17 +82,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 					   SubTransactionId mySubid,
 					   SubTransactionId parentSubid,
@@ -91,24 +100,43 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 						 bool ignore_errors);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 						 PGresult **result);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
+{
+	ConnCacheEntry *entry;
+	ConnCacheKey	key;
+	bool			found;
 
-/*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
- */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
+
+	/*
+	 * Find or create cached entry for requested connection.
+	 */
+	entry = hash_search(ConnectionHash, &key, HASH_ENTER, &found);
+
+	if (!found)
+	{
+		/*
+		 * We need only clear "conn" here; remaining fields will be filled
+		 * later when "conn" is set.
+		 */
+		entry->conn = NULL;
+	}
+
+	return entry;
+}
+
+
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +156,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,24 +163,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
-	/*
-	 * Find or create cached entry for requested connection.
-	 */
-	entry = hash_search(ConnectionHash, &key, HASH_ENTER, &found);
-	if (!found)
-	{
-		/*
-		 * We need only clear "conn" here; remaining fields will be filled
-		 * later when "conn" is set.
-		 */
-		entry->conn = NULL;
-	}
+	entry = GetConnectionCacheEntry(umid);
 
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
@@ -182,6 +192,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping		*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +201,8 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->am_participant_of_ac = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +213,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,16 +229,46 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
 /*
  * Connect to remote server using specified server and user mapping properties.
+ * If the attempt to connect fails, and the caller can handle connection failure
+ * (connection_error_ok = true) return NULL, throw error otherwise.
  */
 static PGconn *
 connect_pg_server(ForeignServer *server, UserMapping *user)
@@ -265,11 +317,22 @@ connect_pg_server(ForeignServer *server, UserMapping *user)
 
 		conn = PQconnectdbParams(keywords, values, false);
 		if (!conn || PQstatus(conn) != CONNECTION_OK)
+		{
+			char	   *connmessage;
+			int			msglen;
+
+			/* libpq typically appends a newline, strip that */
+			connmessage = pstrdup(PQerrorMessage(conn));
+			msglen = strlen(connmessage);
+			if (msglen > 0 && connmessage[msglen - 1] == '\n')
+				connmessage[msglen - 1] = '\0';
+
 			ereport(ERROR,
 					(errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION),
 					 errmsg("could not connect to server \"%s\"",
 							server->servername),
 					 errdetail_internal("%s", pchomp(PQerrorMessage(conn)))));
+		}
 
 		/*
 		 * Check that non-superuser has used password to establish connection;
@@ -414,15 +477,20 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
+	ForeignServer	*server = GetForeignServer(serverid);
 
 	/* Start main transaction if we haven't yet */
 	if (entry->xact_depth <= 0)
 	{
 		const char *sql;
 
+		/* Register the new foreign server if enabled */
+		if (server_uses_twophase_commit(server))
+			entry->am_participant_of_ac = true;
+
 		elog(DEBUG3, "starting remote transaction on connection %p",
 			 entry->conn);
 
@@ -644,193 +712,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 }
 
 /*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow remote transactions that modified anything,
-					 * since it's not very reasonable to hold them open until
-					 * the prepared transaction is committed.  For the moment,
-					 * throw error unconditionally; later we might allow
-					 * read-only cases.  Note that the error will cause us to
-					 * come right back here with event == XACT_EVENT_ABORT, so
-					 * we'll clean up the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot prepare a transaction that modified remote tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
-/*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
 static void
@@ -846,10 +727,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -860,6 +737,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			return;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1193,3 +1074,327 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * The function prepares transaction on foreign server. This function
+ * is called only at the pre-commit phase of the local transaction. Since
+ * we should have the connection to the server that we are interested in
+ * we don't use serverid and userid that are necessary to get user mapping
+ * that is the key of the connection cache.
+ */
+bool
+postgresPrepareForeignTransaction(FdwXactState *state)
+{
+	PgFdwXactState *rstate;
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+	PGresult	*res;
+	StringInfo	command;
+
+	entry = GetConnectionState(state->umid, false, false);
+	//entry = hash_search(ConnectionHash, &(state->umid), HASH_FIND, NULL);
+
+	if (!entry->xact_got_connection || !entry->conn)
+		return true;
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	rstate = (PgFdwXactState *) palloc0(sizeof(PgFdwXactState));
+	rstate->serverid = state->serverid;
+	rstate->userid = state->userid;
+	rstate->umid = state->umid;
+	rstate->conn = entry;
+	state->fdw_state = (void *)rstate;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	if (result)
+		elog(DEBUG1, "prepared foreign transaction on server %u with ID %s",
+			 state->serverid, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/*
+ * The function commits the transactionon foreign server. This
+ * function is called both at the pre-commit phase of the local transaction
+ * when committing and at the end of the local transaction when aborting.
+ * Since we should the connections to the server that involved with the local
+ * transaction we don't use serverid and userid that are necessary to get
+ * user mapping that is the key of connection cache.
+ */
+bool
+postgresCommitForeignTransaction(FdwXactState *state)
+{
+	PgFdwXactState *rstate;
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+	PGresult	*res;
+
+	entry = GetConnectionState(state->umid, false, false);
+
+	if (!entry->xact_got_connection || !entry->conn)
+		return true;
+
+	rstate = (PgFdwXactState *) palloc0(sizeof(PgFdwXactState));
+	rstate->serverid = state->serverid;
+	rstate->userid = state->userid;
+	rstate->umid = state->umid;
+	rstate->conn = entry;
+	state->fdw_state = (void *)rstate;
+
+	/*
+	 * If abort cleanup previously failed for this connection,
+	 * we can't issue any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	/*
+	 * If there were any errors in subtransactions, and we
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+	entry->have_prep_stmt = false;
+	entry->have_error = false;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/*
+ * The function rollbacks the transactionon foreign server. This
+ * function is called both at the pre-commit phase of the local transaction
+ * when committing and at the end of the local transaction when aborting.
+ * Since we should the connections to the server that involved with the local
+ * transaction we don't use serverid and userid that are necessary to get
+ * user mapping that is the key of connection cache.
+ */
+bool
+postgresRollbackForeignTransaction(FdwXactState *state)
+{
+	PgFdwXactState *rstate = (PgFdwXactState *) state->fdw_state;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (rstate)
+		entry = rstate->conn;
+	else
+		entry = GetConnectionCacheEntry(state->umid);
+
+	/*
+	 * In rollback local transaction, not having connection entry means that
+	 * no transaction started. So we can regard it as success.
+	 */
+	if (!entry->xact_got_connection || !entry->conn)
+		return true;
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is already unsalvageable, do only the cleanup
+	 * and don't touch it further.
+	 */
+	if (entry->changing_xact_state)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return true;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+	else
+	{
+		entry->have_prep_stmt = false;
+		entry->have_error = false;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return !abort_cleanup_failure;
+}
+
+bool
+postgresResolveForeignTransaction(FdwXactState *state, bool is_commit)
+{
+	ConnCacheEntry *entry = NULL;
+	StringInfo	command;
+	bool result = false;
+	PGresult	*res;
+
+	entry = GetConnectionState(state->umid, false, false);
+
+	if (!entry->conn)
+		return false;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 state->fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		/*
+		 * The command failed, raise a warning to log the reason of failure.
+		 * We may not be in a transaction here, so raising error doesn't
+		 * help. Even if we are in a transaction, it would be the resolver
+		 * transaction, which will get aborted on raising error, thus
+		 * delaying resolution of other prepared foreign transactions.
+		 */
+		pgfdw_report_error(LOG, res, entry->conn, false, command->data);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * If we tried to COMMIT/ABORT a prepared transaction and the prepared
+		 * transaction was missing on the foreign server, it was probably
+		 * resolved by some other means. Anyway, it should be considered as resolved.
+		 */
+		result = (sqlstate == ERRCODE_UNDEFINED_OBJECT);
+	}
+	else
+		result = true;
+
+	elog(DEBUG1, "%s prepared foreign transaction on server %u with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 state->serverid,
+		 state->fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->am_participant_of_ac = false;
+
+	/*
+	 * Regardless of the event type, we can now mark ourselves as out of the
+	 * transaction.
+	 */
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 21a2ef5..15dadf4 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,15 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -185,16 +207,19 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                                      List of foreign tables
- Schema |   Table    |  Server   |                   FDW options                    | Description 
---------+------------+-----------+--------------------------------------------------+-------------
- public | ft1        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft2        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft4        | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
- public | ft5        | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft6        | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft_pg_type | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
-(6 rows)
+                                         List of foreign tables
+ Schema |      Table       |  Server   |                   FDW options                    | Description 
+--------+------------------+-----------+--------------------------------------------------+-------------
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
+(9 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8650,3 +8675,345 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+\det+
+                                                 List of foreign tables
+ Schema |      Table       |  Server   |                            FDW options                            | Description 
+--------+------------------+-----------+-------------------------------------------------------------------+-------------
+ public | fpagg_tab_p1     | loopback  | (table_name 'pagg_tab_p1')                                        | 
+ public | fpagg_tab_p2     | loopback  | (table_name 'pagg_tab_p2')                                        | 
+ public | fpagg_tab_p3     | loopback  | (table_name 'pagg_tab_p3')                                        | 
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')                             | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1', use_remote_estimate 'true') | 
+ public | ft3              | loopback  | (table_name 'loct3', use_remote_estimate 'true')                  | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')                             | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type')                  | 
+ public | ftprt1_p1        | loopback  | (table_name 'fprt1_p1', use_remote_estimate 'true')               | 
+ public | ftprt1_p2        | loopback  | (table_name 'fprt1_p2')                                           | 
+ public | ftprt2_p1        | loopback  | (table_name 'fprt2_p1', use_remote_estimate 'true')               | 
+ public | ftprt2_p2        | loopback  | (table_name 'fprt2_p2', use_remote_estimate 'true')               | 
+ public | rem1             | loopback  | (table_name 'loc1')                                               | 
+ public | rem2             | loopback  | (table_name 'loc2')                                               | 
+(19 rows)
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+  srvname  
+-----------
+ loopback
+ loopback2
+ loopback3
+(3 rows)
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO on;
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO ft8_twophase VALUES(3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO "S 1"."T 6" VALUES (4);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  4
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(5);
+INSERT INTO "S 1"."T 6" VALUES (5);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  4
+(1 row)
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(8);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Fails, cannot commit the distributed transaction if 2PC-non-capable
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+ERROR:  cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+(4 rows)
+
+-- Disables atomic commit, and success the same case as above.
+SET foreign_twophase_commit TO off;
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+(5 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+(5 rows)
+
+-- Enable atomic commit, again.
+SET foreign_twophase_commit TO on;
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(10);
+INSERT INTO ft8_twophase VALUES(10);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+-- Fails, cannot prepare the transaction if non-supporeted
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(11);
+INSERT INTO ft9_not_twophase VALUES(11);
+PREPARE TRANSACTION 'gx1'; -- error
+ERROR:  cannot prepare a transaction that modified remote tables
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
+SELECT * FROM ft9_not_twophase;
+ c1 
+----
+  1
+  2
+  2
+  4
+  6
+  6
+  9
+  9
+(8 rows)
+
diff --git a/contrib/postgres_fdw/option.c b/contrib/postgres_fdw/option.c
index 6854f1b..1f45b1c 100644
--- a/contrib/postgres_fdw/option.c
+++ b/contrib/postgres_fdw/option.c
@@ -108,7 +108,8 @@ postgres_fdw_validator(PG_FUNCTION_ARGS)
 		 * Validate option value, when we can do so without any context.
 		 */
 		if (strcmp(def->defname, "use_remote_estimate") == 0 ||
-			strcmp(def->defname, "updatable") == 0)
+			strcmp(def->defname, "updatable") == 0 ||
+			strcmp(def->defname, "two_phase_commit") == 0)
 		{
 			/* these accept only boolean values */
 			(void) defGetBoolean(def);
@@ -177,6 +178,8 @@ InitPgFdwOptions(void)
 		/* fetch_size is available on both server and table */
 		{"fetch_size", ForeignServerRelationId, false},
 		{"fetch_size", ForeignTableRelationId, false},
+		/* two phase commit support */
+		{"two_phase_commit", ForeignServerRelationId, false},
 		{NULL, InvalidOid, false}
 	};
 
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index fd20aa9..1135046 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,8 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "catalog/pg_class.h"
@@ -359,6 +361,7 @@ static void postgresGetForeignUpperPaths(PlannerInfo *root,
 							 RelOptInfo *input_rel,
 							 RelOptInfo *output_rel,
 							 void *extra);
+static bool postgresIsTwoPhaseCommitEnabled(Oid serverid);
 
 /*
  * Helper functions
@@ -452,7 +455,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 				  const PgFdwRelationInfo *fpinfo_o,
 				  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -506,10 +508,29 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+	routine->ResolveForeignTransaction = postgresResolveForeignTransaction;
+	routine->IsTwoPhaseCommitEnabled = postgresIsTwoPhaseCommitEnabled;
+
 	PG_RETURN_POINTER(routine);
 }
 
 /*
+ * postgresIsTwoPhaseCommitEnabled
+ */
+static bool
+postgresIsTwoPhaseCommitEnabled(Oid serverid)
+{
+	ForeignServer	*server = GetForeignServer(serverid);
+
+
+	return server_uses_twophase_commit(server);
+}
+
+/*
  * postgresGetForeignRelSize
  *		Estimate # of rows and width of the result of the scan
  *
@@ -1356,7 +1377,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2411,7 +2432,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2704,7 +2725,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								&retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3321,7 +3342,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4108,7 +4129,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4198,7 +4219,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4421,7 +4442,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
@@ -5803,3 +5824,26 @@ find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel)
 	/* We didn't find any suitable equivalence class expression */
 	return NULL;
 }
+
+/*
+ * server_uses_twophase_commit
+ * Returns true if the foreign server is configured to support 2PC.
+ */
+bool
+server_uses_twophase_commit(ForeignServer *server)
+{
+	ListCell		*lc;
+
+	/* Check the options for two phase compliance */
+	foreach(lc, server->options)
+	{
+		DefElem    *d = (DefElem *) lfirst(lc);
+
+		if (strcmp(d->defname, "two_phase_commit") == 0)
+		{
+			return defGetBoolean(d);
+		}
+	}
+	/* By default a server is not 2PC compliant */
+	return false;
+}
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index 70b538e..3526923 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/relation.h"
@@ -110,12 +111,14 @@ typedef struct PgFdwRelationInfo
 	int			relation_index;
 } PgFdwRelationInfo;
 
+typedef struct ConnCacheEntry ConnCacheEntry;
+
 /* in postgres_fdw.c */
 extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -123,6 +126,11 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 				   bool clear, const char *sql);
+extern bool postgresPrepareForeignTransaction(FdwXactState *state);
+extern bool postgresCommitForeignTransaction(FdwXactState *state);
+extern bool postgresRollbackForeignTransaction(FdwXactState *state);
+extern bool postgresResolveForeignTransaction(FdwXactState *state,
+											  bool is_commit);
 
 /* in option.c */
 extern int ExtractConnectionOptions(List *defelems,
@@ -181,6 +189,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 						List *remote_conds, List *pathkeys, bool is_subquery,
 						List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 88c4cb4..2554c9c 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,19 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -2304,7 +2331,6 @@ SELECT t1.a, t2.b FROM fprt1 t1 INNER JOIN fprt2 t2 ON (t1.a = t2.b) WHERE t1.a
 
 RESET enable_partitionwise_join;
 
-
 -- ===================================================================
 -- test partitionwise aggregates
 -- ===================================================================
@@ -2354,3 +2380,126 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+
+\det+
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO on;
+
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO ft8_twophase VALUES(3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO "S 1"."T 6" VALUES (4);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(5);
+INSERT INTO "S 1"."T 6" VALUES (5);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(8);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Fails, cannot commit the distributed transaction if 2PC-non-capable
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Disables atomic commit, and success the same case as above.
+SET foreign_twophase_commit TO off;
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
+
+-- Enable atomic commit, again.
+SET foreign_twophase_commit TO on;
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(10);
+INSERT INTO ft8_twophase VALUES(10);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft7_twophase;
+SELECT * FROM ft8_twophase;
+
+-- Fails, cannot prepare the transaction if non-supporeted
+-- server involved in.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(11);
+INSERT INTO ft9_not_twophase VALUES(11);
+PREPARE TRANSACTION 'gx1'; -- error
+SELECT * FROM ft8_twophase;
+SELECT * FROM ft9_not_twophase;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 54b5e98..f4a9ff5 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -436,6 +436,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
-- 
2.10.5

From 8f9796852b6325037f4ee997fc787a27ab869ce5 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:44:42 +0900
Subject: [PATCH v21 2/4] Support atomic commit among multiple foreign servers.

---
 doc/src/sgml/catalogs.sgml                    |   97 +
 doc/src/sgml/config.sgml                      |  143 +-
 doc/src/sgml/distributed-transaction.sgml     |  157 ++
 doc/src/sgml/fdwhandler.sgml                  |  203 ++
 doc/src/sgml/filelist.sgml                    |    1 +
 doc/src/sgml/func.sgml                        |   51 +
 doc/src/sgml/monitoring.sgml                  |   60 +
 doc/src/sgml/postgres.sgml                    |    1 +
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/fdwxact.c          | 2678 +++++++++++++++++++++++++
 src/backend/access/fdwxact/fdwxact_launcher.c |  641 ++++++
 src/backend/access/fdwxact/fdwxact_resolver.c |  331 +++
 src/backend/access/heap/heapam.c              |   12 -
 src/backend/access/rmgrdesc/Makefile          |    8 +-
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   65 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/Makefile           |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   32 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/foreigncmds.c            |   23 +
 src/backend/executor/execPartition.c          |    4 +
 src/backend/executor/nodeForeignscan.c        |    8 +
 src/backend/executor/nodeModifyTable.c        |   24 +
 src/backend/foreign/foreign.c                 |   43 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   18 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    2 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   80 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  147 ++
 src/include/access/fdwxact_launcher.h         |   32 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   52 +
 src/include/access/resolver_internal.h        |   67 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   23 +
 src/include/foreign/fdwapi.h                  |   18 +-
 src/include/foreign/foreign.h                 |    2 +-
 src/include/pgstat.h                          |    8 +-
 src/include/storage/proc.h                    |   10 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    2 +
 src/test/regress/expected/rules.out           |   12 +
 62 files changed, 5287 insertions(+), 40 deletions(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100755 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/fdwxact_launcher.c
 create mode 100644 src/backend/access/fdwxact/fdwxact_resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 9edba96..391dc7f 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9624,6 +9624,103 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-prepared-fdw-xacts">
+  <title><structname>pg_prepared_fdw_xacts</structname></title>
+
+  <indexterm zone="view-pg-prepared-fdw-xacts">
+   <primary>pg_prepared_fdw_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_prepared_fdw_xacts</structname> displays
+   information about foreign transactions that are currently prepared on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="fdw-transaction-managements"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_prepared_xacts</structname> contains one row per prepared
+   foreign transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_prepared_fdw_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>transaction</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Transaction id that this foreign transaction associates with
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server that this foreign server is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction: <literal>prepared</literal>, <literal>committing</literal>, <literal>aborting</literal> or <literal>unknown</literal>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 7554cba..557b3f2 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3611,7 +3611,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -7827,6 +7826,148 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophase_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign transaction
+         to be resolved before the command returns a "success" indication to the client.
+         Valid values are <literal>required</literal>, <literal>prefer</literal> and
+         <literal>disabled</literal>. The default setting is <literal>disabled</literal>.
+         When <literal>disabled</literal>, there can be risk of database consistency among
+         distributed transaction if some foreign server crashes during committing the
+         distributed transaction. When set to <literal>required</literal> the distributed
+         transaction requires that all written servers can use two-phase commit protocol.
+         That is, the transaction fails if any of servers returns <literal>false</literal>
+         from <function>IsTwoPhaseCommitEnabled</function> or does not support transaction
+         management callback routines(described in
+         <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction requires
+         two-phase commit protocol where available but without failing when it is not
+         available.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero value to
+         set this parameter either <literal>required</literal> or <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one transaction
+         is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism.  You should set this value to
+         zero only if you set <varname>max_foreign_transaction_resolvers</varname> as
+         much as databases you have. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000..5143499
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,157 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction Management</title>
+
+ <para>
+  This chapter explains what distributed transaction management is, and how it can be configured
+  in PostgreSQL.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit is an operation that applies a set of changes as a single operation
+   globally. <productname>PostgreSQL</productname> provides a way to perform a transaction
+   with foreign resources using <literal>Foreign Data Wrapper</literal>. Using the
+   <productname>PostgreSQL</productname>'s atomic commit ensures that all changes
+   on foreign servers end in either commit or rollback using the transaction callback
+   routines (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs Two-phase commit protocol, which is a
+    type of atomic commitment protocol (ACP). Using Two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed transaction manager
+    prepares all transaction on the foreign servers if two-phase commit is required.
+    Two-phase commit is required only if the transaction modifies data on two or more
+    servers including the local server itself and user requests it by
+    <xref linkend="guc-foreign-twophase-commit"/>. If all preparations on foreign servers
+    got successful go to the next step. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes over rollback, then rollback all transactions
+    on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the transaction
+    locally. Any failure happens in this step <productname>PostgreSQL</productname> changes
+    over rollback, then rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign Transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Foreign Transaction Resolution</title>
+
+   <para>
+    Foreign transaction resolutions are performed by foreign transaction resolver process.
+    They commit all prepared transaction on foreign servers if the coordinator received
+    an agreement message from all foreign server during the first step. On the other hand,
+    if any foreign server failed to prepare the transaction, it rollbacks all prepared
+    transactions.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions on one
+    database of the coordinator side. On failure during resolution, they retries to
+    resolve after <varname>foreign_transaction_resolution_interval</varname>.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>In-doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit or rollback
+    using two-phase commit protocol. However, if the second phase fails for whatever reason
+    the transaction becomes in-doubt. The transactions becomes in-doubt in the following
+    situations:
+
+   <itemizedlist>
+    <listitem>
+     <para>
+      A local <productname>PostgreSQL</productname> server crashes during atomic commit
+      operation.
+     </para>
+    </listitem>
+
+    <listitem>
+     <para>
+      A local <productname>PostgreSQL</productname> server got a cancellation by user during
+      atomic commit.
+     </para>
+    </listitem>
+   </itemizedlist>
+
+   In-doubt transactions are automatically handled by foreign transaction resolver process
+   when there is no online transaction requesting resolutions.
+   <function>pg_resolve_fdw_xact</function> provides a way to resolve transactions on foreign
+   servers manually that participated the distributed transaction manually.
+   </para>
+
+   <para>
+    The atomic commit operation is crash-safe. The being processed foreign transactions at
+    crash are processed by a foreign transaction resolvers as an in-doubt transaction
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-fdwxact-resolver-view"><literal>pg_stat_fdwxact_resolver</literal></link>
+    view. This view contains one row for every foreign Transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 4ce88dd..3da13c9 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1390,6 +1390,118 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     If an FDW wishes to support <firstterm>atomic commit</firstterm>
+     (as described in <xref linkend="fdw-transaction-managements"/>), it must call the
+     registrasaction function <function>FdwXactRegisterForeignTransaction</function>
+     and provide the following callback functions:
+    </para>
+
+    <para>
+<programlisting>
+bool
+PrepareForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if atomic commit is required.
+    Returning <literal>true</literal> means that preparing the foreign
+    transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Commit the not-prepared transaction on the foreign server.
+    This function is called at the pre-commit phase of local
+    transaction if atomic commit is not required. The atomic
+    commit is not required either when we modified data on
+    only one server including the local server or when userdoesn't
+    request atomic commit by <xref linkend="guc-foreign-twophase-commit"/>.
+    Returning <literal>true</literal> means that commit the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Rollback a not-prepared transaction on the foreign server.
+    This function is called at the end of local transaction after
+    rollbacked locally either when user requested rollback or when
+    any error occurs during the transaction. This function could
+    be called recursively if any error occurs during rollback the
+    foreign transaction for whatever reason. You need to track
+    recursion and prevent this function from being called infinitely.
+    Returning <literal>true</literal> means that rollback the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+ResolvePreparedForeignTransaction(FdwXactResolveState *state,
+                                  bool is_commit);
+</programlisting>
+    Commit or rollback the prepared transaction on the foreign server.
+    When <varname>is_commit</varname> is true, it indicates that the foreign
+    transaction should be committed. Otherwise the foreign transaction should
+    be aborted.
+    This function normally is called by the foreign transaction resolver
+    process but can also be called by <function>pg_resovle_fdw_xacts</function>
+    function. In the resolver process, this function is called either
+    when a backend requests the resolver process to resolve a distributed
+    transaction after prepared, or when a database has dangling
+    transactions. Returning <literal>true</literal> means that resolving
+    the foreign transaction got successful.
+    In abort case, please note that the prepared transaction identified
+    by <varname>state->fdwxact_id</varname> might not exist on the foreign
+    server. If you failed to resolve the foreign transaction due to undefined
+    object error (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) you should
+   regards it as success and return <literal>true</literal>.
+    </para>
+    <para>
+<programlisting>
+bool
+IsTwoPhaseCommitEnabled(Oid serverid);
+</programlisting>
+    Return <literal>true</literal> if the foreign server identified by
+    <literal>serverid</literal> is capable of two-phase commit protocol.
+    This function is called at commit time once.
+    Return <literal>false</literal> indicates that the current transaction
+    cannot use atomic commit even if atomic commit is requested by user.
+    </para>
+
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1835,4 +1947,95 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+    <title>Transaction managements for Foreign Data Wrappers</title>
+
+    <para>
+     <productname>PostgreSQL</productname> foreign transaction manager
+     allows FDWs to read and write data on foreign server within a transaction while
+     maintaining atomicity of the foreign data (aka atomic commit). Using
+     atomic commit, it guarantees that a distributed transaction is committed
+     or rollbacked on all participants foreign
+     server.  To achieve atomic commit, <productname>PostgreSQL</productname>
+     employees two-phase commit protocol, which is a type of atomic commitment
+     protocol. Every FDW that wish to support atomic commit
+     is required to support the transaction management callback routines:
+     <function>PrepareForeignTransaction</function>,
+     <function>CommitForeignTransaction</function>,
+     <function>RollbackForeignTransaction</function>,
+     <function>ResolveForeignTransaction</function>,
+     <function>IsTwoPhaseCommitEnabled</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+     Tranasction of foreign server that supports these callback routines is
+     managed by <productname>PostgreSQL</productname>'s distributed  transaction
+     manager. Each transaction management callbacks are called at appropriate time.
+    </para>
+
+    <para>
+     The information in <literal>FdwXactState</literal> can be used to identify
+     foreign servers. <literal>state-&gt;fdw_state</literal> is a <type>void</type>
+     pointer that is available for FDW transaction functions to store Information
+     relevant to the particular foreign server.  It is useful for passing
+     information forward from <function>PrepareForeignTransaction</function> and/or
+     <function>CommitTransaciton</function> to
+     <function>RollbackForeignTransaction</function>, there by avoiding recalculation.
+     Note that since <function>ResolveForeignTransaction</function> is called
+     idependently from these callback routines, the information is not passed to
+     <function>ResolverForeignTransaction</function>.
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling <function>PrepareForeignTransaction</function>
+     if two-phase commit protocol is required. Two-phase commit is required only if
+     the transaction modified data on more than one servers including the local
+     server itself and user requests atomic commit. <productname>PostgreSQL</productname>
+     can commit locally and go to the next step if and only if all preparing foreign
+     transactions got successful. If two-phase commit is not required, the foreign
+     transaction manager commits each transaction calling
+     <function>CommitForeignTransaction</function> and then commit locally.
+     If any failure happens or user requests to cancel during the pre-commit phase
+     the distributed Transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function> for not-prepared foreign
+     servers, and then rollback locally. The prepared foreign servers are rollbacked
+     by a foreign transaction resolver process.
+    </para>
+
+    <para>
+     Once committed locally, the distributed transaction must be committed. The
+     prepared foreign transaction will be committed by foreign transaction resolver
+     process.
+    </para>
+
+    <para>
+     When two-phase commit is required, after committed locally, the transaction
+     commit will wait for all prepared foreign transaction to be committed before
+     completetion. One foreign transaction resolver process is responsible for
+     foreign transaction resolution on a database.
+     <function>ResolverForeignTransaction</function> is called by the foreign
+     transaction resolver process when resolution.
+     <function>ResolveForeignTransaction</function> is also be called
+     when user executes <function>pg_resovle_fdw_xact</function> function.
+    </para>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 48ac14a..38d6fcb 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -49,6 +49,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 96d4541..3690df1 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -20825,6 +20825,57 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-fdw-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_fdw_xacts</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_fdw_xacts</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function search for foreign transaction
+        matching the arguments and resolves then. This function won't resolve
+        a foreign transaction which is in progress, or one that is locked by some
+        other backend.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_fdw_xact</function>
+        except it remove foreign transaction entry without resolving.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index add7145..00f0030 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -332,6 +332,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_fdw_xact_resolver</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-fdwxact-resolver-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1194,6 +1202,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
+        <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
         </row>
@@ -1409,6 +1429,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
+        <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
          <entry>Waiting during base backup when throttling activity.</entry>
@@ -2218,6 +2242,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-fdwxact-resolver-view" xreflabel="pg_stat_fdw_xact_resolver">
+   <title><structname>pg_stat_fdw_xact_resolver</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index 0070603..c10e21f 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -164,6 +164,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index bd93a6a..4a1ebdc 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  tablesample transam
+			  tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000..9ddbb14
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o fdwxact_resolver.o fdwxact_launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100755
index 0000000..1f270db
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2678 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL distributed transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers automically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * When a foreign data wrapper starts transaction on a foreign server that
+ * is capable of two-phase commit protocol, foreign data wrappers registers
+ * the foreign transaction using function FdwXactRegisterForeignTransaction()
+ * in order to participate to a group for atomic commit. Participants are
+ * identified by oid of foreign server and user. When the foreign transaction
+ * begins to modify data the executor marks it as modified using
+ * FdwXactMarkForeignTransactionModified().
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. After committing or rolling back locally, we
+ * notify the resolver process and tell it to commit or roll back those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail when an ERROR
+ * after already committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * waiters each time we receive a request. We have two queues: the active
+ * queue and the retry queue. The backend is inserted to the active queue at
+ * first, and then it is moved to the retry queue by the resolver process if
+ * the resolution fails. The backends in the retry queue are processed at
+ * interval of foreign_transaction_resolution_retry_interval.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or more
+ * servers including itself. In other case, all foreign transactions are
+ * committed during pre-commit.
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. dangling
+ * transaction). Dangling transactions are processed by the resolve process
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * 	* On PREPARE redo we add the foreign transaction to FdwXactCtl->fdw_xacts.
+ *	  We set fdw_xact->inredo to true for such entries.
+ *	* On Checkpoint redo, we iterate through FdwXactCtl->fdw_xacts entries that
+ *	  have set fdw_xact->inredo true and are behind the redo_horizon. We save
+ *    them to disk and then set fdw_xact->ondisk to true.
+ *	* On COMMIT and ABORT we delete the entry from FdwXactCtl->fdw_xacts.
+ *	  If fdw_xact->ondisk is true, we delete the corresponding file from
+ *	  the disk as well.
+ *  * RecoverFdwXacts loads all foreign transaction entries from disk into
+ *    memory at server startup.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Is atomic commit requested by user? */
+#define IsAtomicCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+#define IsAtomicCommitRequested() \
+	(IsAtomicCommitEnabled() && \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED))
+
+#define FDW_XACT_ACTION_COMMIT	 		0x01
+#define FDW_XACT_ACTION_TWOPHASE_COMMIT 0x02
+
+/* Structure to bundle the foreign transaction participant */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in global entry. NULL if
+	 * this foreign transaction is registered but not inserted
+	 * yet.
+	 */
+	FdwXact		fdw_xact;
+	char		*fdw_xact_id;
+
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	bool		modified;					/* true if modified the data on server */
+	bool		twophase_commit_enabled;	/* true if the server can execute
+											 * two-phase commit protocol */
+	void			*fdw_state;				/* fdw-private state */
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact;
+	CommitForeignTransaction_function	commit_foreign_xact;
+	RollbackForeignTransaction_function	rollback_foreign_xact;
+	IsTwoPhaseCommitEnabled_function	is_twophase_commit_enabled;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit.
+ * This list has only foreign servers that support atomic commit FDW
+ * API regardless of their configuration.
+ */
+static List *FdwXactAtomicCommitParticipants = NIL;
+static bool FdwXactAtomicCommitReady = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDW_XACTS_DIR "pg_fdw_xact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDW_XACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDW_XACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+static void FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, bool modified);
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactCommitForeignTransaction(FdwXactParticipant *fdw_part);
+static bool FdwXactResolveForeignTransaction(FdwXactState *state, FdwXact fdwxact,
+											 int elevel);
+static void FdwXactComputeRequiredXmin(void);
+static bool FdwXactAtomicCommitRequired(void);
+static void FdwXactQueueInsert(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid, bool give_warnings);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							  bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid, Oid serverid,
+								  Oid userid, XLogRecPtr insert_start_lsn,
+								  bool give_warnings);
+static List *get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						   bool need_lock);
+static FdwXact get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								bool need_lock);
+static FdwXact get_all_fdw_xacts(int *length);
+static FdwXact insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							   Oid umid, char *fdw_xact_id);
+static char *generate_fdw_xact_identifier(TransactionId xid, Oid serverid, Oid userid);
+static void remove_fdw_xact(FdwXact fdw_xact);
+static FdwXactState *create_fdw_xact_state(void);
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+int foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+/*
+ * Remember accessed foreign server. This function is called by executor when
+ * it begins to access foreign server. If FDW of the foreign server supports
+ * atomic commit API, it is registered as a transaction participant of distributed
+ * transaction.
+ */
+void
+FdwXactMarkForeignServerAccessed(Relation rel, int flags, bool modified)
+{
+	FdwRoutine			*fdwroutine;
+	ListCell   			*lc;
+	Oid					serverid;
+	Oid					userid;
+
+	/* Quick return if atomic commit is not enabled */
+	if (!IsAtomicCommitEnabled())
+		return;
+
+	/* Do nothing in EXPLAIN (no ANALYZE) case */
+	if (flags && EXEC_FLAG_EXPLAIN_ONLY)
+		return;
+
+	serverid = GetForeignServerIdByRelId(RelationGetRelid(rel));
+	fdwroutine  = GetFdwRoutineByRelId(RelationGetRelid(rel));
+
+	/*
+	 * If the being modified foreign server doesn't have the atomic commit API
+	 * we don't manage the foreign transaction in the distributed transaction
+	 * manager.
+	 */
+	if (fdwroutine->IsTwoPhaseCommitEnabled == NULL)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		return;
+	}
+
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	foreach(lc, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	FdwXactRegisterForeignTransaction(serverid, userid, modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign server identified by given server id must support atomic
+ * commit APIs. Registered foreign transaction are managed by foreign
+ * transaction manager until the end of the transaction.
+ */
+static void
+FdwXactRegisterForeignTransaction(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant	*fdw_part;
+	ForeignServer 		*foreign_server;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	FdwRoutine			*fdw_routine;
+	MemoryContext		old_ctx;
+	char				*fdwxact_id;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	/*
+	 * Participants information is needed at the end of a transaction, where
+	 * system cache are not available. Save it in TopTransactionContext
+	 * beforehand so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	foreign_server = GetForeignServer(serverid);
+	fdw = GetForeignDataWrapper(foreign_server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+	user_mapping = GetUserMapping(userid, serverid);
+
+	/* Make sure that the FDW has transaction handlers */
+	if (!fdw_routine->PrepareForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function provided for preparing foreign transaction for FDW %s",
+						fdw->fdwname)));
+	if (!fdw_routine->CommitForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to commit a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+	if (!fdw_routine->RollbackForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to rollback a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+
+	/* Generate an unique identifier */
+	if (fdw_routine->GetPrepareId)
+	{
+		char *id;
+		int fdwxact_id_len = 0;
+
+		id = fdw_routine->GetPrepareId(GetTopTransactionId(),
+											   foreign_server->serverid,
+											   user_mapping->userid,
+											   &fdwxact_id_len);
+
+		if (!id)
+			ereport(ERROR,
+					(errcode(ERRCODE_UNDEFINED_OBJECT),
+					 (errmsg("foreign transaction identifier is not provided"))));
+
+		/* Check length of foreign transaction identifier */
+		id[fdwxact_id_len] = '\0';
+		if (fdwxact_id_len > NAMEDATALEN)
+			ereport(ERROR,
+					(errcode(ERRCODE_NAME_TOO_LONG),
+					 errmsg("foreign transaction identifer \"%s\" is too long",
+							id),
+					 errdetail("foreign transaction identifier must be less than %d characters.",
+							   NAMEDATALEN)));
+
+		fdwxact_id = pstrdup(id);
+	}
+	else
+		fdwxact_id = generate_fdw_xact_identifier(GetTopTransactionId(),
+												  serverid, userid);
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdw_xact_id = fdwxact_id;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdw_xact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->twophase_commit_enabled = true; /* by default, will be changed at pre-commit phase */
+	fdw_part->fdw_state = NULL;
+	fdw_part->prepare_foreign_xact = fdw_routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact = fdw_routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact = fdw_routine->RollbackForeignTransaction;
+	fdw_part->is_twophase_commit_enabled = fdw_routine->IsTwoPhaseCommitEnabled;
+
+	/* Add this foreign transaction to the participants list */
+	FdwXactAtomicCommitParticipants = lappend(FdwXactAtomicCommitParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+
+	return;
+}
+
+/*
+ * FdwXactShmemSize
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdw_xacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * FdwXactShmemInit
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdw_xacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->freeFdwXacts = NULL;
+		FdwXactCtl->numFdwXacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdw_xacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdw_xacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdw_xacts[cnt].status = FDW_XACT_INITIAL;
+			fdw_xacts[cnt].fxact_free_next = FdwXactCtl->freeFdwXacts;
+			FdwXactCtl->freeFdwXacts = &fdw_xacts[cnt];
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * PreCommit_FdwXacts
+ *
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_atomic_commit;
+	ListCell	*lc;
+	ListCell	*next;
+	ListCell	*prev = NULL;
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsAtomicCommitRequested())
+		return;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	need_atomic_commit = FdwXactAtomicCommitRequired();
+
+	/*
+	 * If 'require' case, we require all modified server have to be capable of
+	 * two-phase commit protocol.
+	 */
+	if (need_atomic_commit &&
+		foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	/*
+	 * Commit transactions on foreign servers.
+	 *
+	 * Committed transactions are removed from FdwXactAtomicCommitParticipants
+	 * so that the later preparation can process only servers that requires to be commit
+	 * using two-phase commit protocol.
+	 */
+	for (lc = list_head(FdwXactAtomicCommitParticipants); lc != NULL; lc = next)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool can_commit = false;
+
+		next = lnext(lc);
+
+		if (!need_atomic_commit || !fdw_part->modified)
+		{
+			/*
+			 * We can commit not-modified servers and when the atomic commit is not
+			 * required.
+			 */
+			can_commit = true;
+		}
+		else if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER &&
+				 !fdw_part->twophase_commit_enabled)
+		{
+			/* Also in 'prefer' case, non-2pc-capable servers can be committed */
+			can_commit = true;
+		}
+
+		if (can_commit)
+		{
+			/* Commit the foreign transaction */
+			FdwXactCommitForeignTransaction(fdw_part);
+
+			/* Delete it from the participant list */
+			FdwXactAtomicCommitParticipants =
+				list_delete_cell(FdwXactAtomicCommitParticipants, lc, prev);
+		}
+
+		prev = lc;
+	}
+
+	/*
+	 * If only one participant of all participants is modified, we can commit it.
+	 * This can avoid to use two-phase commit for only one server in the 'prefer' case
+	 * where the transaction has one 2pc-capable modified server and some modified
+	 * servers.
+	 */
+	if (list_length(FdwXactAtomicCommitParticipants) == 1 &&
+		(MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) == 0)
+	{
+		Assert(foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER);
+		FdwXactCommitForeignTransaction(linitial(FdwXactAtomicCommitParticipants));
+		list_free(FdwXactAtomicCommitParticipants);
+		return;
+	}
+
+	FdwXactPrepareForeignTransactions();
+	/* keep FdwXactparticipantsForAC until the end of transaction */
+}
+
+/*
+ * FdwXactPrepareForeignTransactions
+ *
+ * Prepare all foreign transaction participants.  This function creates a prepared
+ * participants chain each time when we prepared a foreign transaction. The prepared
+ * participants chain is used to access all participants of distributed transaction
+ * quickly. If any one of them fails to prepare, we change over aborts.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	FdwXactState *state;
+	ListCell   *lcell;
+	FdwXact		prev_fdwxact = NULL;
+
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	state = create_fdw_xact_state();
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXact		fdwxact;
+
+		/*
+		 * Insert the foreign transaction entry. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(GetTopTransactionId(), fdw_part);
+
+		state->serverid = fdw_part->server->serverid;
+		state->userid = fdw_part->usermapping->userid;
+		state->umid = fdw_part->usermapping->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		/*
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal). During abort processing,
+		 * we might try to resolve a never-prepared transaction, and get an error.
+		 * This is fine as long as the FDW provides us unique prepared transaction
+		 * identifiers.
+		 */
+		if (!fdw_part->prepare_foreign_xact(state))
+		{
+			/* Failed to prepare, change over aborts */
+			ereport(ERROR,
+					(errmsg("could not prepare transaction on foreign server %s",
+							fdw_part->server->servername)));
+		}
+
+		/* Keep fdw_state until end of transaction */
+		fdw_part->fdw_state = state->fdw_state;
+
+		/* Preparation is success, update its status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdw_part->fdw_xact->status = FDW_XACT_PREPARED;
+		fdw_part->fdw_xact = fdwxact;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Create a prepared participants chain, which is link-ed FdwXact entries
+		 * involving with this transaction.
+		 */
+		if (prev_fdwxact)
+		{
+			/* Append others to the tail */
+			Assert(fdwxact->fxact_next == NULL);
+			prev_fdwxact->fxact_next = fdwxact;
+		}
+	}
+}
+
+/*
+ * Commit the given foreign transaction.
+ */
+void
+FdwXactCommitForeignTransaction(FdwXactParticipant *fdw_part)
+{
+	FdwXactState *state;
+
+	state = create_fdw_xact_state();
+	state->serverid = fdw_part->server->serverid;
+	state->userid = fdw_part->usermapping->userid;
+	state->umid = fdw_part->usermapping->umid;
+	fdw_part->fdw_state = (void *) state;
+
+	if (!fdw_part->commit_foreign_xact(state))
+		ereport(ERROR,
+				(errmsg("could not commit foreign transaction on server %s",
+						fdw_part->server->servername)));
+}
+
+/*
+ * FdwXactInsertFdwXactEntry
+ *
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and will
+ * be persisted to the disk under pg_fdw_xact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fxact;
+	FdwXactOnDiskData	*fxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact = insert_fdw_xact(MyDatabaseId, xid, fdw_part->server->serverid,
+							fdw_part->usermapping->userid,
+							fdw_part->usermapping->umid, fdw_part->fdw_xact_id);
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->held_by = MyBackendId;
+	fdw_part->fdw_xact = fxact;
+	LWLockRelease(FdwXactLock);
+
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdw_xact_id);
+	data_len = data_len + strlen(fdw_part->fdw_xact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fxact_file_data->dbid = MyDatabaseId;
+	fxact_file_data->local_xid = xid;
+	fxact_file_data->serverid = fdw_part->server->serverid;
+	fxact_file_data->userid = fdw_part->usermapping->userid;
+	fxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fxact_file_data->fdw_xact_id, fdw_part->fdw_xact_id,
+		   strlen(fdw_part->fdw_xact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fxact_file_data, data_len);
+	fxact->insert_end_lsn = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_INSERT);
+	XLogFlush(fxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact->valid = true;
+	LWLockRelease(FdwXactLock);
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fxact_file_data);
+	return fxact;
+}
+
+/*
+ * insert_fdw_xact
+ *
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				Oid umid, char *fdw_xact_id)
+{
+	int i;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		fxact = FdwXactCtl->fdw_xacts[i];
+		if (fxact->dbid == dbid &&
+			fxact->local_xid == xid &&
+			fxact->serverid == serverid &&
+			fxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->freeFdwXacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fxact = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fxact->fxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->numFdwXacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts++] = fxact;
+
+	fxact->held_by = InvalidBackendId;
+	fxact->dbid = dbid;
+	fxact->local_xid = xid;
+	fxact->serverid = serverid;
+	fxact->userid = userid;
+	fxact->umid = umid;
+	fxact->insert_start_lsn = InvalidXLogRecPtr;
+	fxact->insert_end_lsn = InvalidXLogRecPtr;
+	fxact->valid = false;
+	fxact->ondisk = false;
+	fxact->inredo = false;
+	memcpy(fxact->fdw_xact_id, fdw_xact_id, strlen(fdw_xact_id) + 1);
+
+	return fxact;
+}
+
+/*
+ * remove_fdw_xact
+ *
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdw_xact(FdwXact fdw_xact)
+{
+	int			cnt;
+
+	Assert(fdw_xact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Search the slot where this entry resided */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		if (FdwXactCtl->fdw_xacts[cnt] == fdw_xact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (cnt >= FdwXactCtl->numFdwXacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdw_xact->local_xid, fdw_xact->serverid, fdw_xact->userid)));
+
+	/* Remove the entry from active array */
+	FdwXactCtl->numFdwXacts--;
+	FdwXactCtl->fdw_xacts[cnt] = FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts];
+
+	/* Put it back into free list */
+	fdw_xact->fxact_free_next = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fdw_xact;
+
+	/* Reset informations */
+	fdw_xact->status = FDW_XACT_INITIAL;
+	fdw_xact->held_by = InvalidBackendId;
+	fdw_xact->fxact_next = NULL;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdw_xact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdw_xact->serverid;
+		record.dbid = fdw_xact->dbid;
+		record.xid = fdw_xact->local_xid;
+		record.userid = fdw_xact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdw_xact_remove));
+		recptr = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true and set FdwXactAtomicCommitReady to true if we require atomic commit.
+ * It is required if the transaction modified data on two or more servers including
+ * local node itself. This function also checks for each server if two-phase commit
+ * is enabled or not.
+ */
+static bool
+FdwXactAtomicCommitRequired(void)
+{
+	ListCell*	lc;
+	int			nserverswritten = 0;
+
+	if (!IsAtomicCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		/* Check if the foreign server is capable of two-phase commit protocol */
+		if (fdw_part->is_twophase_commit_enabled(fdw_part->server->serverid))
+			fdw_part->twophase_commit_enabled = true;
+		else if (fdw_part->modified)
+			MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	/* Atomic commit is required if we modified data on two or more participants */
+	if (nserverswritten <= 1)
+		return false;
+
+	FdwXactAtomicCommitReady = true;
+	return true;
+}
+
+bool
+FdwXactIsAtomicCommitReady(void)
+{
+	return FdwXactAtomicCommitReady;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * ForgetAllFdwXactParticipants
+ *
+ * Reset all the foreign transaction entries that this backend registered.
+ * If the foreign transaction has the corresponding FdwXact entry, resetting
+ * the held_by field means to leave that entry in unresolved state. If we
+ * leaves any entries, we update the oldest xmin of unresolved transaction
+ * so that transaction status of dangling transaction are not truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_lefts = 0;
+
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(cell, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+
+		/* Skip if didn't register FdwXact entry yet */
+		if (fdw_part->fdw_xact == NULL)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in
+		 * FdwXactAtomicCommitParticipants could be used by other backend before we
+		 * forget in case where the resolver process removes the FdwXact entry
+		 * and other backend reuses it before we forget. So we need to check
+		 * if the entries are still associated with the transaction.
+		 */
+		if (fdw_part->fdw_xact->held_by == MyBackendId)
+		{
+			fdw_part->fdw_xact->held_by = InvalidBackendId;
+			n_lefts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Update the oldest local transaction of unresolved distributed
+	 * transaction if we leaved any FdwXact entries.
+	 */
+	if (n_lefts > 0)
+		FdwXactComputeRequiredXmin();
+
+	FdwXactAtomicCommitParticipants = NIL;
+}
+
+/*
+ * AtProcExit_FdwXact
+ *
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+/*
+ * Wait for foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDW_XACT_NOT_WAITING and then change
+ * that state to FDW_XACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDW_XACT_WAIT_COMPLETE once foreign transactions are resolved.
+ * This backend then resets its state to FDW_XACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue and changes the state to FDW_XACT_WAITING_RETRY.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+	ListCell	*lc;
+	List		*fdwxact_participants = NIL;
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsAtomicCommitRequested())
+		return;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDW_XACT_NOT_WAITING);
+
+	if (FdwXactAtomicCommitParticipants != NIL)
+	{
+		/*
+		 * If we're waiting for foreign transactions to be resolved that
+		 * we've prepared just before, use the participants list.
+		 */
+		Assert(MyPgXact->xid == wait_xid);
+		fdwxact_participants = FdwXactAtomicCommitParticipants;
+	}
+	else
+	{
+		/*
+		 * Get participants list from the global array. This is required (1)
+		 * when we're waiting for foreign transactions to be resolved that
+		 * is part of a local prepared transaction that is marked as prepared
+		 * during running, or (2) when we resolve the PREPARE'd distributed
+		 * transaction after restart.
+		 */
+		fdwxact_participants = get_fdw_xacts(MyDatabaseId, wait_xid,
+											 InvalidOid, InvalidOid, true);
+	}
+
+	/* Exit if we found no foreign transaction to resolve */
+	if (fdwxact_participants == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(lc, fdwxact_participants)
+	{
+		FdwXact fdw_xact = (FdwXact) lfirst(lc);
+
+		/* Don't overwrite status if fate has been determined */
+		if (fdw_xact->status == FDW_XACT_PREPARED)
+			fdw_xact->status = (is_commit ?
+								FDW_XACT_COMMITTING_PREPARED :
+								FDW_XACT_ABORTING_PREPARED);
+	}
+
+	/* Set backend status and enqueue itself to the active queue*/
+	MyProc->fdwXactState = FDW_XACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	FdwXactQueueInsert();
+	LWLockRelease(FdwXactLock);
+
+	/* Launch a resolver process if not yet, or wake it up */
+	fdwxact_maybe_launch_resolver(false);
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDW_XACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDW_XACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDW_XACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+
+	/*
+	 * Forget the list of locked entries, also means that the entries
+	 * that could not resolved are remained as dangling transactions.
+	 */
+	ForgetAllFdwXactParticipants();
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Acquire FdwXactLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Insert MyProc into the tail of FdwXactActiveQueue.
+ */
+static void
+FdwXactQueueInsert(void)
+{
+	SHMQueueInsertBefore(&(FdwXactRslvCtl->FdwXactActiveQueue),
+						 &(MyProc->fdwXactLinks));
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * Resolve one distributed transaction. The target distributed transaction
+ * is fetched from either the active queue or the retry queue and its participants
+ * are fetched from either the global array.
+ *
+ * Release the waiter and return true if we resolved the all of the foreign
+ * transaction participants. On failure, we move the FdwXactLinks entry to the
+ * retry queue from the active queue, and raise an error and exit.
+ */
+bool
+FdwXactResolveDistributedTransaction(Oid dbid, bool is_active)
+{
+	FdwXactState	*state;
+	ListCell		*lc;
+	ListCell		*next;
+	PGPROC			*waiter = NULL;
+	List			*participants;
+	SHM_QUEUE		*target_queue;
+
+	if (is_active)
+		target_queue = &(FdwXactRslvCtl->FdwXactActiveQueue);
+	else
+		target_queue = &(FdwXactRslvCtl->FdwXactRetryQueue);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/* Fetch a waiter from beginning of the queue */
+	while ((waiter = (PGPROC *) SHMQueueNext(target_queue, target_queue,
+											 offsetof(PGPROC, fdwXactLinks))) != NULL)
+	{
+		/* Found a waiter */
+		if (waiter->databaseId == dbid)
+			break;
+	}
+
+	/* If no waiter, there is no job */
+	if (!waiter)
+	{
+		LWLockRelease(FdwXactLock);
+		return false;
+	}
+
+	Assert(TransactionIdIsValid(waiter->fdwXactWaitXid));
+
+	state = create_fdw_xact_state();
+	participants = get_fdw_xacts(dbid, waiter->fdwXactWaitXid, InvalidOid,
+								 InvalidOid, false);
+	LWLockRelease(FdwXactLock);
+
+	/* Resolve all foreign transactions one by one */
+	for (lc = list_head(participants); lc != NULL; lc = next)
+	{
+		FdwXact fdwxact = (FdwXact) lfirst(lc);
+
+		CHECK_FOR_INTERRUPTS();
+
+		next = lnext(lc);
+
+		state->serverid = fdwxact->serverid;
+		state->userid = fdwxact->userid;
+		state->umid = fdwxact->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		PG_TRY();
+		{
+			FdwXactResolveForeignTransaction(state, fdwxact, ERROR);
+		}
+		PG_CATCH();
+		{
+			/* Re-insert the waiter to the retry queue */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDW_XACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				SHMQueueInsertBefore(&(FdwXactRslvCtl->FdwXactRetryQueue),
+									 &(waiter->fdwXactLinks));
+				waiter->fdwXactState = FDW_XACT_WAITING_RETRY;
+			}
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved a foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+	}
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter
+	 * could already be detached if user cancelled to wait before
+	 * resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId	wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDW_XACT_WAIT_COMPLETE;
+
+		/* Wake up the waiter only when we have set state and removed from queue */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc xid %u", wait_xid);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	return true;
+}
+
+/*
+ * Resolve all dangling foreign transactions on the given database. Get
+ * all dangling foreign transactions from shmem global array and resolve
+ * them one by one.
+ */
+void
+FdwXactResolveAllDanglingTransactions(Oid dbid)
+{
+	List		*dangling_fdwxacts = NIL;
+	ListCell	*cell;
+	bool		n_resolved = 0;
+	int			i;
+
+	Assert(OidIsValid(dbid));
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/*
+	 * Walk over the global array to make the list of dangling transactions
+	 * of which corresponding local transaction is on the given database.
+	 */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fxact = FdwXactCtl->fdw_xacts[i];
+
+		/*
+		 * Append the fdwxact entry on the given database to the list if
+		 * it's handled by nobody and the corresponding local transaction
+		 * is not part of the prepared transaction.
+		 */
+		if (fxact->dbid == dbid &&
+			fxact->held_by == InvalidBackendId &&
+			!TwoPhaseExists(fxact->local_xid))
+			dangling_fdwxacts = lappend(dangling_fdwxacts, fxact);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/* Return if there is no foreign transaction we need to resolve */
+	if (dangling_fdwxacts == NIL)
+		return;
+
+	foreach(cell, dangling_fdwxacts)
+	{
+		FdwXact fdwxact = (FdwXact) lfirst(cell);
+		FdwXactState *state;
+
+		state = create_fdw_xact_state();
+		state->serverid = fdwxact->serverid;
+		state->userid = fdwxact->userid;
+		state->umid = fdwxact->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		FdwXactResolveForeignTransaction(state, fdwxact, ERROR);
+
+		n_resolved++;
+	}
+
+	list_free(dangling_fdwxacts);
+
+	elog(DEBUG2, "resolved %d dangling foreign xacts", n_resolved);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ *
+ * In commit case, we have already prepared transactions on the foreign
+ * servers during pre-commit. And that prepared transactions will be
+ * resolved by the resolver process. So we don't do anything about the
+ * foreign transaction.
+ *
+ * In abort case, user requested rollback or we changed over rollback
+ * due to error during commit. To close current foreign transaction anyway
+ * we call rollback API to every foreign transaction. If we raised an error
+ * during preparing and came to here, it's possible that some entries of
+ * FdwXactParticipants already registered its FdwXact entry. If there is
+ * we leave them as dangling transaction and ask the resolver process to
+ * process them.
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		int left_fdwxacts = 0;
+		FdwXactState *state = create_fdw_xact_state();
+
+		foreach (lcell, FdwXactAtomicCommitParticipants)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * Count FdwXact entries that we registered to shared memory array
+			 * in this transaction.
+			 */
+			if (fdw_part->fdw_xact)
+			{
+				/*
+				 * The status of foreign transaction must be either preparing
+				 * or prepared. In any case, since we have registered FdwXact
+				 * entry we leave them to the resolver process. For the preparing
+				 * state, since the foreign transaction might not close yet we
+				 * fall through and call rollback API. For the prepared state,
+				 * since the foreign transaction has closed we don't need to do
+				 * anything.
+				 */
+				Assert(fdw_part->fdw_xact->status == FDW_XACT_PREPARING ||
+					   fdw_part->fdw_xact->status == FDW_XACT_PREPARED);
+
+				left_fdwxacts++;
+				if (fdw_part->fdw_xact->status == FDW_XACT_PREPARED)
+					continue;
+			}
+
+			state->serverid = fdw_part->server->serverid;
+			state->userid = fdw_part->usermapping->userid;
+			state->umid = fdw_part->usermapping->umid;
+			state->fdw_state = fdw_part->fdw_state;
+
+			/*
+			 * Rollback all current foreign transaction. Since we're rollbacking
+			 * the transaction it's too late even if we raise an error here.
+			 * So we log it as warning.
+			 */
+			if (!fdw_part->rollback_foreign_xact(state))
+				ereport(WARNING,
+						(errmsg("could not abort transaction on server \"%s\"",
+								fdw_part->server->servername)));
+		}
+
+		/* If we left some FdwXact entries, ask the resolver process */
+		if (left_fdwxacts > 0)
+		{
+			ereport(WARNING,
+					(errmsg("might have left %u foreign transactions in in-doubt status",
+							left_fdwxacts)));
+			fdwxact_maybe_launch_resolver(true);
+		}
+	}
+
+	ForgetAllFdwXactParticipants();
+	FdwXactAtomicCommitReady = false;
+}
+
+/*
+ * AtPrepare_FdwXacts
+ *
+ * If there are foreign servers involved in the transaction, this function
+ * prepares transactions on those servers.
+ *
+ * Note that it can happen that the transaction aborts after we prepared part
+ * of participants. In this case since we can change to abort we cannot forget
+ * FdwXactAtomicCommitParticipants here. These are processed by the resolver process
+ * during aborting, or at EOXact_FdwXacts.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (!IsAtomicCommitEnabled())
+		return;
+
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (FdwXactAtomicCommitRequired() &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_T_R_INTEGRITY_CONSTRAINT_VIOLATION),
+				 errmsg("can not prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * FdwXactResolveForeignTransaction
+ *
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * handler routine. The foreign transaction can be a dangling transaction
+ * that is not interested by nobody. If the fate of foreign transaction is
+ * not determined yet, it'sdetermined according to the status of corresponding
+ * local transaction.
+ *
+ * If the resolution is successful, remove the foreign transaction entry from
+ * the shared memory and also remove the corresponding on-disk file.
+ */
+static bool
+FdwXactResolveForeignTransaction(FdwXactState *state, FdwXact fdwxact,
+								 int elevel)
+{
+	ForeignServer		*server;
+	ForeignDataWrapper	*fdw;
+	FdwRoutine			*fdw_routine;
+	bool		is_commit;
+	bool		ret;
+
+	Assert(fdwxact);
+
+	/*
+	 * Determine whether we commit or abort this foreign transaction.
+	 */
+	if (fdwxact->status == FDW_XACT_COMMITTING_PREPARED)
+		is_commit = true;
+	else if (fdwxact->status == FDW_XACT_ABORTING_PREPARED)
+		is_commit = false;
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	else if (TransactionIdDidCommit(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_COMMITTING_PREPARED;
+		is_commit = true;
+	}
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_ABORTING_PREPARED;
+		is_commit = false;
+	}
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+		is_commit = false;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction
+	 * state is neither committing or aborting. This should not
+	 * happen because we cannot determine to do commit or abort for
+	 * foreign transaction associated with the in-progress local
+	 * transaction.
+	 */
+	else
+		ereport(ERROR,
+				(errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid)));
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	/* Resolve the foreign transaction */
+	Assert(fdw_routine->ResolveForeignTransaction);
+
+	ret = fdw_routine->ResolveForeignTransaction(state, is_commit);
+
+	if (!ret)
+	{
+		ereport(elevel,
+				(errmsg("could not %s a prepared foreign transaction on server \"%s\"",
+						is_commit ? "commit" : "rollback", server->servername),
+				 errdetail("local transaction id is %u, connected by user id %u",
+						   fdwxact->local_xid, fdwxact->userid)));
+	}
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdw_xact(fdwxact);
+	LWLockRelease(FdwXactLock);
+
+	return ret;
+}
+
+static FdwXactState *
+create_fdw_xact_state(void)
+{
+	FdwXactState *state;
+
+	state = palloc(sizeof(FdwXactState));
+	state->serverid = InvalidOid;
+	state->userid = InvalidOid;
+	state->umid = InvalidOid;
+	state->fdwxact_id = NULL;
+	state->fdw_state = NULL;
+
+	return state;
+}
+
+/*
+ * Return one FdwXact entry that matches to given arguments, otherwise
+ * return NULL. Since this function search FdwXact entry by unique key
+ * all arguments should be valid.
+ */
+static FdwXact
+get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				 bool need_lock)
+{
+	List	*fdw_xact_list;
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, need_lock);
+
+	/* Could not find entry */
+	if (fdw_xact_list == NIL)
+		return NULL;
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdw_xact_list) == 1);
+
+	return (FdwXact) linitial(fdw_xact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdw_xact_exists(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdw_xact_list;
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, true);
+
+	return fdw_xact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_prepared_fdw_xacts.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdw_xacts(int *length)
+{
+	List		*all_fdw_xacts;
+	ListCell	*lc;
+	FdwXact		fdw_xacts;
+	int			num_fdw_xacts = 0;
+
+	Assert(length != NULL);
+
+	/* Get all entries */
+	all_fdw_xacts = get_fdw_xacts(InvalidOid, InvalidTransactionId,
+								  InvalidOid, InvalidOid, true);
+
+	if (all_fdw_xacts == NIL)
+	{
+		*length = 0;
+		return NULL;
+	}
+
+	fdw_xacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdw_xacts));
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdw_xacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdw_xacts + num_fdw_xacts, fx,
+			   sizeof(FdwXactData));
+		num_fdw_xacts++;
+	}
+
+	*length = num_fdw_xacts;
+	list_free(all_fdw_xacts);
+
+	return fdw_xacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return
+ * NIL.
+ */
+static List*
+get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			  bool need_lock)
+{
+	int i;
+	List	*fdw_xact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact	fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool	matches = true;
+
+		/* xid */
+		if (xid != InvalidTransactionId && xid != fdw_xact->local_xid)
+			matches = false;
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdw_xact->dbid != dbid)
+			matches = false;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdw_xact->serverid)
+			matches = false;
+
+		/* userid */
+		if (OidIsValid(userid) && fdw_xact->userid != userid)
+			matches = false;
+
+		/* Append it if matched */
+		if (matches)
+			fdw_xact_list = lappend(fdw_xact_list, fdw_xact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdw_xact_list;
+}
+
+/*
+ * fdw_xact_redo
+ * Apply the redo log for a foreign transaction.
+ */
+void
+fdw_xact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDW_XACT_REMOVE)
+	{
+		xl_fdw_xact_remove *record = (xl_fdw_xact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier with in the form
+ * of "fx_<random number>_<xid>_<serverid>_<userid> whose length is always
+ * less than NAMEDATALEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+generate_fdw_xact_identifier(TransactionId xid, Oid serverid, Oid userid)
+{
+	char*	fdw_xact_id;
+
+	fdw_xact_id = (char *)palloc0(FDW_XACT_ID_MAX_LEN * sizeof(char));
+
+	snprintf(fdw_xact_id, FDW_XACT_ID_MAX_LEN, "%s_%ld_%u_%d_%d",
+			 "fx", Abs(random()), xid, serverid, userid);
+	fdw_xact_id[strlen(fdw_xact_id)] = '\0';
+
+	return fdw_xact_id;
+}
+
+/*
+ * CheckPointFdwXact
+ *
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * In order to avoid disk I/O while holding a light weight lock, the function
+ * first collects the files which need to be synced under FdwXactLock and then
+ * syncs them after releasing the lock. This approach creates a race condition:
+ * after releasing the lock, and before syncing a file, the corresponding
+ * foreign transaction entry and hence the file might get removed. The function
+ * checks whether that's true and ignores the error if so.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdw_xacts = 0;
+
+	/* Quick get-away, before taking lock */
+	if (max_prepared_foreign_xacts <= 0)
+		return;
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/* Another quick, before we allocate memory */
+	if (FdwXactCtl->numFdwXacts <= 0)
+	{
+		LWLockRelease(FdwXactLock);
+		return;
+	}
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		FdwXact		fxact = FdwXactCtl->fdw_xacts[cnt];
+
+		if ((fxact->valid || fxact->inredo) &&
+			!fxact->ondisk &&
+			fxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fxact->dbid, fxact->local_xid,
+								fxact->serverid, fxact->userid,
+								buf, len);
+			fxact->ondisk = true;
+			fxact->insert_start_lsn = InvalidXLogRecPtr;
+			fxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdw_xacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDW_XACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdw_xacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdw_xacts,
+							 serialized_fdw_xacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDW_XACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDW_XACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * ProcessFdwXactBuffer
+ *
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid = ShmemVariableCache->nextXid;
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(insert_start_lsn != InvalidXLogRecPtr);
+
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid, true);
+		if (buf == NULL)
+		{
+			ereport(WARNING,
+					(errmsg("removing corrupt fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+			return NULL;
+		}
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return thecontents in
+ * a structure allocated in-memory. Otherwise return NULL. The structure can
+ * be later freed by the caller.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				bool give_warnings)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+	{
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not stat FDW transaction state file \"%s\": %m",
+							path)));
+		return NULL;
+	}
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdw_xact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+	{
+		CloseTransientFile(fd);
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+		return NULL;
+	}
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+	{
+		CloseTransientFile(fd);
+		return NULL;
+	}
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_READ);
+	if (read(fd, buf, stat.st_size) != stat.st_size)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not read FDW transaction state file \"%s\": %m",
+					  path)));
+		return NULL;
+	}
+
+	pgstat_report_wait_end();
+	CloseTransientFile(fd);
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+	{
+		pfree(buf);
+		return NULL;
+	}
+
+	/* Check if the contents is an expected data */
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fxact_file_data->dbid  != dbid ||
+		fxact_file_data->serverid != serverid ||
+		fxact_file_data->userid != userid ||
+		fxact_file_data->local_xid != xid)
+	{
+		ereport(WARNING,
+			(errmsg("invalid foreign transaction state file \"%s\"",
+					path)));
+		CloseTransientFile(fd);
+		pfree(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+/*
+ * PrescanFdwXacts
+ *
+ * Scan the all foreign transactions directory for oldest active transaction.
+ * This is run during database startup, after we completed reading WAL.
+ * ShmemVariableCache->nextXid has been set to one more than the highest XID
+ * for which evidence exists in WAL.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	TransactionId nextXid = ShmemVariableCache->nextXid;
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+		 strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			TransactionId local_xid;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/*
+			 * Remove a foreign prepared transaction file corresponding to an
+			 * XID, which is too new.
+			 */
+			if (TransactionIdFollowsOrEquals(local_xid, nextXid))
+			{
+				ereport(WARNING,
+						(errmsg("removing future foreign prepared transaction file \"%s\"",
+								clde->d_name)));
+				RemoveFdwXactFile(dbid, local_xid, serverid, userid, true);
+				continue;
+			}
+
+			if (TransactionIdPrecedesOrEquals(local_xid, oldestActiveXid))
+				oldestActiveXid = local_xid;
+		}
+	}
+
+	FreeDir(cldir);
+	return oldestActiveXid;
+}
+
+/*
+ * restoreFdwXactData
+ *
+ * Scan pg_fdw_xact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid, bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * FdwXactRedoAdd
+ *
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fxact = insert_fdw_xact(fxact_data->dbid, fxact_data->local_xid,
+							fxact_data->serverid, fxact_data->userid,
+							fxact_data->umid, fxact_data->fdw_xact_id);
+
+	/*
+	 * Set status as preparing, since we do not know the xact status
+	 * right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->insert_start_lsn = start_lsn;
+	fxact->insert_end_lsn = end_lsn;
+	fxact->inredo = true;	/* added in redo */
+	fxact->valid = false;
+	fxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * FdwXactRedoRemove
+ *
+ * Remove the corresponding fdw_xact entry from FdwXactCtl.
+ * Also remove fdw_xact file if a foreign transaction was saved
+ * via an earlier checkpoint.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdw_xact(dbid, xid, serverid, userid,
+							   false);
+
+	if (fdwxact == NULL)
+		return;
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdw_xact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(int *newval, void **extra, GucSource source)
+{
+	ForeignTwophaseCommitLevel newForeignTwophaseCommitLevel = *newval;
+
+		/* Parameter check */
+	if (newForeignTwophaseCommitLevel > FOREIGN_TWOPHASE_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdw_xacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_prepared_fdw_xacts(PG_FUNCTION_ARGS)
+{
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdw_xacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdw_xacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(6, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdw_xacts = get_all_fdw_xacts(&num_fdw_xacts);
+		status->num_xacts = num_fdw_xacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdw_xact = &status->fdw_xacts[status->cur_xact++];
+		Datum		values[6];
+		bool		nulls[6];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdw_xact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdw_xact->dbid);
+		values[1] = TransactionIdGetDatum(fdw_xact->local_xid);
+		values[2] = ObjectIdGetDatum(fdw_xact->serverid);
+		values[3] = ObjectIdGetDatum(fdw_xact->userid);
+		switch (fdw_xact->status)
+		{
+			case FDW_XACT_PREPARING:
+				xact_status = "prepared";
+				break;
+			case FDW_XACT_COMMITTING_PREPARED:
+				xact_status = "committing";
+				break;
+			case FDW_XACT_ABORTING_PREPARED:
+				xact_status = "aborting";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		/* should this be really interpreted by FDW */
+		values[5] = PointerGetDatum(cstring_to_text_with_len(fdw_xact->fdw_xact_id,
+															 strlen(fdw_xact->fdw_xact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXactState *state;
+	UserMapping		*usermapping;
+	FdwXact			fdwxact;
+	bool			ret;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, true);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	usermapping = GetUserMapping(userid, serverid);
+
+	state = create_fdw_xact_state();
+	state->serverid = serverid;
+	state->userid = userid;
+	state->umid = usermapping->umid;
+
+	ret = FdwXactResolveForeignTransaction(state, fdwxact, LOG);
+
+	PG_RETURN_BOOL(ret);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, false);
+	if (fdwxact == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("could not find foreign transaction entry"))));
+
+	remove_fdw_xact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_VOID();
+}
diff --git a/src/backend/access/fdwxact/fdwxact_launcher.c b/src/backend/access/fdwxact/fdwxact_launcher.c
new file mode 100644
index 0000000..39f351b
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact_launcher.c
@@ -0,0 +1,641 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * There is a shared memory area where the information of resolver process
+ * is stored. Requesting of starting new resolver process by backend process
+ * is done via that shared memory area. Note that the launcher is assuming
+ * that there is no more than one starting request for a database.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/fdwxact_launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launcher_sigusr2(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid, int slot);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+Datum pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS);
+
+/*
+ * Wake up the launcher process to retry launch. This is used by
+ * the resolver process is being stopped.
+ */
+void
+FdwXactLauncherWakeupToRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request resolution. This is
+ * used by the backend process.
+ */
+void
+FdwXactLauncherWakeupToRequest(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->FdwXactActiveQueue));
+		SHMQueueInit(&(FdwXactRslvCtl->FdwXactRetryQueue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR1: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a foreign_xact_resolution_retry_interval
+		 * but always try to start by the backend request.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool launched;
+
+			ResetLatch(MyLatch);
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher launch",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver worker
+ * if not running yet. A foreign transaction resolver worker is responsible
+ * for resolution of foreign transaction that are registered on a database.
+ * So if a resolver worker already is launched, we don't need to launch new
+ * one.
+ */
+void
+fdwxact_maybe_launch_resolver(bool ignore_error)
+{
+	FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->pid != InvalidPid &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/*
+	 * If we found the resolver for my database, we don't need to launch new
+	 * one but wake running worker up.
+	 */
+	if (found)
+	{
+		SetLatch(resolver->latch);
+
+		elog(DEBUG1, "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		return;
+	}
+
+	/* Looking for unused resolver slot */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	/*
+	 * However if there are no more free worker slots, inform user about it before
+	 * exiting.
+	 */
+	if (!found)
+	{
+		LWLockRelease(FdwXactResolverLock);
+
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+		return;
+	}
+
+	Assert(resolver->pid == InvalidPid);
+
+	/* Found a new resolver process */
+	resolver->dbid = MyDatabaseId;
+	resolver->in_use = true;
+
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Wake up launcher */
+	FdwXactLauncherWakeupToRequest();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid' at 'slot' if given. If slot is negative value we find an unused slot.
+ * Note that caller must hold FdwXactResolverLock in exclusive mode.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid, int slot)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int launch_slot = slot;
+
+	/* If slot number is invalid, we find an unused slot */
+	if (launch_slot < 0)
+	{
+		int i;
+
+		for (i = 0; i < max_foreign_xact_resolvers; i++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+			if (resolver->in_use && resolver->dbid == dbid)
+				return;
+
+			if (!resolver->in_use)
+			{
+				launch_slot = i;
+				break;
+			}
+		}
+	}
+
+	/* No unused found */
+	if (launch_slot < 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[launch_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_main_arg = Int32GetDatum(launch_slot);
+	bgw.bgw_notify_pid = (Datum) 0;
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch all foreign transaction resolvers that are required by backend process
+ * but not running. Return true if we launch any resolver.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	int i, j;
+	int num_launches = 0;
+	int num_unused_slots = 0;
+	int num_dbs = 0;
+	bool launched = false;
+	Oid	*dbs_to_launch;
+	Oid	*dbs_having_worker = palloc0(sizeof(Oid) * max_foreign_xact_resolvers);
+
+	/*
+	 * Launch resolver workers on the databases that are requested
+	 * by backend processes while looking unused slots.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* Remember unused worker slots */
+		if (!resolver->in_use)
+		{
+			num_unused_slots++;
+			continue;
+		}
+
+		/* Remember databases that are having a resolve worker, fall through */
+		if (OidIsValid(resolver->dbid))
+			dbs_having_worker[num_dbs++] = resolver->dbid;
+
+		/* Launch the backend-requested worker */
+		if (resolver->in_use &&
+			OidIsValid(resolver->dbid) &&
+			resolver->pid == InvalidPid)
+		{
+			fdwxact_launch_resolver(resolver->dbid, i);
+			launched = true;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* quick exit if no unused slot */
+	if (num_unused_slots == 0)
+		return launched;
+
+	/*
+	 * Launch the stopped resolver on the database that has unresolved
+	 * foreign transaction but doesn't have any resolver. Scanning
+	 * all FdwXact entries could take time but it's harmless for the
+	 * relaunch case.
+	 */
+	dbs_to_launch = (Oid *) palloc(sizeof(Oid) * num_unused_slots);
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool found = false;
+
+		/* unused slot is full */
+		if (num_launches > num_unused_slots)
+			break;
+
+		for (j = 0; j < num_dbs; j++)
+		{
+			if (dbs_having_worker[j] == fdw_xact->dbid)
+			{
+				found = true;
+				break;
+			}
+		}
+
+		/* Register the database if any resolvers aren't working on that */
+		if (!found)
+			dbs_to_launch[num_launches++] = fdw_xact->dbid;
+	}
+
+	/* Launch resolver process for a database at any worker slot */
+	for (i = 0; i < num_launches; i++)
+	{
+		fdwxact_launch_resolver(dbs_to_launch[i], -1);
+		launched = true;
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+
+/*
+ * Returns activity of foreign transaction resolvers, including pids, the number
+ * of tasks and the last resolution time.
+ */
+Datum
+pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/fdwxact_resolver.c b/src/backend/access/fdwxact/fdwxact_resolver.c
new file mode 100644
index 0000000..0b754da
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact_resolver.c
@@ -0,0 +1,331 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for every databases.
+ *
+ * A resolver process continues to resolve foreign transactions on a database
+ * It resolves two types of foreign transactions: on-line foreign transaction
+ * and dangling foreign transaction. The on-line foreign transaction is a
+ * foreign transaction that a concurrent backend process is waiting for
+ * resolution. The dangling transaction is a foreign transaction that corresponding
+ * distributed transaction ended up in in-doubt state. A resolver process
+ * doesn' exit as long as there is at least one unresolved foreign transaction
+ * on the database even if the timeout has come.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/fdwxact_resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+
+//static MemoryContext ResolveContext = NULL;
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FdwXactRslvLoop(void);
+static long FdwXactRslvComputeSleepTime(TimestampTz now);
+static void FdwXactRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+	FdwXactLauncherWakeupToRetry();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	TIMESTAMP_NOBEGIN(MyFdwXactResolver->last_resolved_time);
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FdwXactRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FdwXactRslvLoop(void)
+{
+	TimestampTz last_retry_time = 0;
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time;
+		bool		resolved;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		/* Resolve one distributed transaction */
+		StartTransactionCommand();
+		resolved = FdwXactResolveDistributedTransaction(MyDatabaseId, true);
+		CommitTransactionCommand();
+
+		now = GetCurrentTimestamp();
+
+		/* Update my state */
+		if (resolved)
+			MyFdwXactResolver->last_resolved_time = now;
+
+		if (TimestampDifferenceExceeds(last_retry_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			StartTransactionCommand();
+			resolved = FdwXactResolveDistributedTransaction(MyDatabaseId, false);
+			CommitTransactionCommand();
+
+			last_retry_time = GetCurrentTimestamp();
+
+			/* Update my state */
+			if (resolved)
+				MyFdwXactResolver->last_resolved_time = last_retry_time;
+		}
+
+		/* Check for fdwxact resolver timeout */
+		FdwXactRslvCheckTimeout(now);
+
+		/*
+		 * If we have resolved any distributed transaction we go the next
+		 * without both resolving dangling transaction and sleeping because
+		 * there might be other on-line transactions waiting to be resolved.
+		 */
+		if (!resolved)
+		{
+			/* Resolve dangling transactions as mush as possible */
+			StartTransactionCommand();
+			FdwXactResolveAllDanglingTransactions(MyDatabaseId);
+			CommitTransactionCommand();
+
+			sleep_time = FdwXactRslvComputeSleepTime(now);
+
+			MemoryContextResetAndDeleteChildren(resolver_ctx);
+			MemoryContextSwitchTo(TopMemoryContext);
+
+			rc = WaitLatch(MyLatch,
+						   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+						   sleep_time,
+						   WAIT_EVENT_FDW_XACT_RESOLVER_MAIN);
+
+			if (rc & WL_POSTMASTER_DEATH)
+				proc_exit(1);
+		}
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FdwXactRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	/*
+	 * Reached to the timeout. We exit if there is no more both pending on-line
+	 * transactions and dangling transactions.
+	 */
+	if (!fdw_xact_exists(InvalidTransactionId, MyDatabaseId, InvalidOid,
+						 InvalidOid))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyFdwXactResolver->dbid))));
+		CommitTransactionCommand();
+
+		fdwxact_resolver_detach();
+		proc_exit(0);
+	}
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. Return the sleep time
+ * in milliseconds, -1 means that we reached to the timeout and should exits
+ */
+static long
+FdwXactRslvComputeSleepTime(TimestampTz now)
+{
+	static TimestampTz	wakeuptime = 0;
+	long	sleeptime;
+	long	sec_to_timeout;
+	int		microsec_to_timeout;
+
+	if (now >= wakeuptime)
+		wakeuptime = TimestampTzPlusMilliseconds(now,
+												 foreign_xact_resolution_retry_interval);
+
+	/* Compute relative time until wakeup. */
+	TimestampDifference(now, wakeuptime,
+						&sec_to_timeout, &microsec_to_timeout);
+
+	sleeptime = sec_to_timeout * 1000 + microsec_to_timeout / 1000;
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index c2db19b..fb63471 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2629,10 +2629,6 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 		heap_freetuple(heaptup);
 	}
 
-	/* Make note that we've wrote on non-temprary relation */
-	if (RelationNeedsWAL(relation))
-		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
-
 	return HeapTupleGetOid(tup);
 }
 
@@ -3457,10 +3453,6 @@ l1:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
-	/* Make note that we've wrote on non-temprary relation */
-	if (RelationNeedsWAL(relation))
-		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
-
 	return HeapTupleMayBeUpdated;
 }
 
@@ -4411,10 +4403,6 @@ l2:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
-	/* Make note that we've wrote on non-temprary relation */
-	if (RelationNeedsWAL(relation))
-		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
-
 	bms_free(hot_attrs);
 	bms_free(proj_idx_attrs);
 	bms_free(key_attrs);
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index 5514db1..742e825 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -8,9 +8,9 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
-	   gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
-	   mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
-	   smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdwxactdesc.o \
+	genericdesc.o  gindesc.o gistdesc.o hashdesc.o heapdesc.o \
+	logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \
+	seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000..7061bba
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,65 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdw_xactdesc.c
+ *		PostgreSQL distributed transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdw_xactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdw_xact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		FdwXactOnDiskData *fdw_insert_xlog = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_insert_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_insert_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_insert_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_insert_xlog->local_xid);
+		/* TODO: This should be really interpreted by each FDW */
+
+		/*
+		 * TODO: we also need to assess whether we want to add this
+		 * information
+		 */
+		appendStringInfo(buf, " foreign transaction info: %s",
+						 fdw_insert_xlog->fdw_xact_id);
+	}
+	else
+	{
+		xl_fdw_xact_remove *fdw_remove_xlog = (xl_fdw_xact_remove *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_remove_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_remove_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_remove_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_remove_xlog->xid);
+	}
+
+}
+
+const char *
+fdw_xact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDW_XACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDW_XACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 00741c7..4a9ab3d 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -112,14 +112,16 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_prepared_xacts=%d max_locks_per_xact=%d "
 						 "wal_level=%s wal_log_hints=%s "
-						 "track_commit_timestamp=%s",
+						 "track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_prepared_xacts,
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47..f15c83a 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -12,9 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
-	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
-	xact.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o generic_xlog.o multixact.o \
+	parallel.o rmgr.o slru.o subtrans.o timeline.o transam.o twophase.o \
+	twophase_rmgr.o varsup.o xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9368b56..8b360b1 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -9,6 +9,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
 #include "access/generic_xlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 3942734..bc4e109 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -844,6 +845,35 @@ TwoPhaseGetGXact(TransactionId xid)
 }
 
 /*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
+/*
  * TwoPhaseGetDummyProc
  *		Get the dummy backend ID for prepared transaction specified by XID
  *
@@ -2316,6 +2346,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2375,6 +2411,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 8c1621d..9dca0f5 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1131,6 +1132,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1139,6 +1141,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsAtomicCommitReady();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1177,12 +1180,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1340,6 +1344,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -1990,6 +2002,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2146,6 +2161,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2233,6 +2249,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2422,6 +2440,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2627,6 +2646,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 62fc418..8e18eea 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
 #include "access/subtrans.h"
@@ -5261,6 +5262,7 @@ BootStrapXLOG(void)
 	ControlFile->MaxConnections = MaxConnections;
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6348,6 +6350,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -6872,14 +6877,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdw_xact, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -7071,7 +7077,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7577,6 +7586,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7895,6 +7905,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9211,6 +9224,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9644,7 +9658,8 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9676,6 +9691,7 @@ XLogReportParameters(void)
 		ControlFile->MaxConnections = MaxConnections;
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9881,6 +9897,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -10079,6 +10096,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->MaxConnections = xlrec.MaxConnections;
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 53ddc59..c27cd5f 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -291,6 +291,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_prepared_fdw_xacts AS
+       SELECT * FROM pg_prepared_fdw_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
 	l.objoid, l.classoid, l.objsubid,
@@ -773,6 +776,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_fdwxact_resolvers AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_fdwxact_resolver() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index e5dd995..dac1e3a 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
@@ -1093,6 +1094,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1407,6 +1420,16 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
 	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srv->serverid,
+						useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
+	/*
 	 * Do the deletion
 	 */
 	object.classId = UserMappingRelationId;
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index 0bcb237..058bc0a 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "catalog/partition.h"
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_type.h"
@@ -749,7 +750,10 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+		FdwXactMarkForeignServerAccessed(partRelInfo->ri_RelationDesc, 0, true);
+	}
 
 	MemoryContextSwitchTo(oldContext);
 
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 2ec7fcb..4578bc0 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,10 +226,16 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+
+	}
 	else
 		fdwroutine->BeginForeignScan(scanstate, eflags);
 
+	FdwXactMarkForeignServerAccessed(scanstate->ss.ss_currentRelation,
+									 eflags, node->operation != CMD_SELECT);
+
 	return scanstate;
 }
 
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 528f587..66c3699 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "commands/trigger.h"
@@ -44,6 +45,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "storage/bufmgr.h"
@@ -485,6 +487,10 @@ ExecInsert(ModifyTableState *mtstate,
 								HEAP_INSERT_SPECULATIVE,
 								NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self),
 												   estate, true, &specConflict,
@@ -530,6 +536,10 @@ ExecInsert(ModifyTableState *mtstate,
 								estate->es_output_cid,
 								0, NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self),
@@ -722,6 +732,11 @@ ldelete:;
 							 true /* wait for commit */ ,
 							 &hufd,
 							 changingPart);
+
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case HeapTupleSelfUpdated:
@@ -1210,6 +1225,11 @@ lreplace:;
 							 estate->es_crosscheck_snapshot,
 							 true /* wait for commit */ ,
 							 &hufd, &lockmode);
+
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case HeapTupleSelfUpdated:
@@ -2321,6 +2341,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 															 fdw_private,
 															 i,
 															 eflags);
+
+			/* Mark this transaction modified data on the foreign server */
+			FdwXactMarkForeignServerAccessed(resultRelInfo->ri_RelationDesc,
+											 eflags, true);
 		}
 
 		resultRelInfo++;
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index a0bcc04..b2097ad 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -155,6 +155,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMapping - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index d2b695e..b722b9a 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -15,6 +15,8 @@
 #include <unistd.h>
 
 #include "libpq/pqsignal.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 42bccce..5116369 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3484,6 +3484,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3678,6 +3684,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -3893,6 +3902,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDW_XACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 688f462..883ad85 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -896,6 +898,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -971,12 +977,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index afb4972..960fd6a 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -154,6 +154,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDW_XACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 0c86a58..c5610ee 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -150,6 +152,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
 		size = add_size(size, BackendRandomShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -270,6 +274,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 	SyncScanShmemInit();
 	AsyncShmemInit();
 	BackendRandomShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 908f62d..cc578b2 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -90,6 +90,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -245,6 +247,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1323,6 +1326,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	volatile TransactionId replication_slot_xmin = InvalidTransactionId;
 	volatile TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	volatile TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1384,6 +1388,7 @@ GetOldestXmin(Relation rel, int flags)
 	/* fetch into volatile var while ProcArrayLock is held */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1434,6 +1439,15 @@ GetOldestXmin(Relation rel, int flags)
 		result = replication_slot_xmin;
 
 	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDW_XACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
+	/*
 	 * After locks have been released and defer_cleanup_age has been applied,
 	 * check whether we need to back up further to make logical decoding
 	 * possible. We need to do so if we're computing the global limit (rel =
@@ -3016,6 +3030,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index e6025ec..a42d06e 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -50,3 +50,5 @@ OldSnapshotTimeMapLock				42
 BackendRandomLock					43
 LogicalRepWorkerLock				44
 CLogTruncationLock					45
+FdwXactLock					46
+FdwXactResolverLock			47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 6f9aaa5..8e55dad 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -398,6 +399,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* initialize fields for fdw xact */
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -799,6 +804,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index a3b9757..48f3c59 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -2994,6 +2996,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 2317e8b..7651352 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/transam.h"
@@ -378,6 +379,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 };
 
 /*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
+/*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
  */
@@ -659,6 +679,10 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2235,6 +2259,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4056,6 +4126,16 @@ static struct config_enum ConfigureNamesEnum[] =
 	},
 
 	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+			gettext_noop("Sets the usage of two-phase commit protocol for distributed transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
+	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
 			NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 4e61bc6..88cdc85 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -121,6 +121,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -287,6 +289,20 @@
 
 
 #------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
+#------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index ad06e8e..ca3eb62 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index ab5cb7f..609578c 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -209,6 +209,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdw_xact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 895a51f..7df88e0 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -306,6 +306,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_worker_processes);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 6fb403a..6d867c8 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -730,6 +730,7 @@ GuessControlValues(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -957,6 +958,7 @@ RewriteControlFile(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* Contents are protected with a CRC */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca..b616cea 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000..0928f4c
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,147 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL distributed transaction manager
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDW_XACT_H
+#define FDW_XACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+#define	FDW_XACT_NOT_WAITING		0
+#define	FDW_XACT_WAITING			1
+#define	FDW_XACT_WAITING_RETRY		2
+#define	FDW_XACT_WAIT_COMPLETE		3
+
+#define FdwXactEnabled() (max_prepared_foreign_xacts > 0)
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDW_XACT_ID_MAX_LEN 200
+
+/* Enum to track the status of prepared foreign transaction */
+typedef enum
+{
+	FDW_XACT_INITIAL,
+	FDW_XACT_PREPARING,					/* foreign transaction is being prepared */
+	FDW_XACT_PREPARED,					/* foreign transaction is prepared */
+	FDW_XACT_COMMITTING_PREPARED,		/* foreign prepared transaction is to
+										 * be committed */
+	FDW_XACT_ABORTING_PREPARED, /* foreign prepared transaction is to be
+								 * aborted */
+} FdwXactStatus;
+
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER,	/* use twophase commit where available */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support twophase
+								 * commit */
+} ForeignTwophaseCommitLevel;
+
+/* Shared memory entry for a prepared or being prepared foreign transaction */
+typedef struct FdwXactData *FdwXact;
+
+typedef struct FdwXactData
+{
+	FdwXact		fxact_free_next;	/* Next free FdwXact entry */
+	FdwXact		fxact_next;			/* Pointer to the neext FdwXact entry accosiated
+									 * with the same transaction */
+	Oid				dbid;			/* database oid where to find foreign server
+									 * and user mapping */
+	TransactionId	local_xid;		/* XID of local transaction */
+	Oid				serverid;		/* foreign server where transaction takes place */
+	Oid				userid;			/* user who initiated the foreign transaction */
+	Oid				umid;
+	FdwXactStatus 	status;			/* The state of the foreign transaction. This
+									 * doubles as the action to be taken on this entry. */
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;		/* XLOG offset of inserting this entry start */
+	XLogRecPtr	insert_end_lsn;		/* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN];		/* prepared transaction identifier */
+} FdwXactData;
+
+/* Shared memory layout for maintaining foreign prepared transaction entries. */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		freeFdwXacts;
+
+	/* Number of valid foreign transaction entries */
+	int			numFdwXacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdw_xacts[FLEXIBLE_ARRAY_MEMBER];		/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+typedef struct FdwXactState
+{
+	Oid		serverid;
+	Oid		userid;
+	Oid		umid;
+	char	*fdwxact_id;
+	void	*fdw_state;		/* foreign-data wrapper can keep state here */
+} FdwXactState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern bool fdw_xact_exists(TransactionId xid, Oid dboid, Oid serverid,
+				Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern void PreCommit_FdwXacts(void);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern bool FdwXactResolveDistributedTransaction(Oid dbid, bool is_active);
+extern void FdwXactResolveAllDanglingTransactions(Oid dbid);
+extern bool FdwXactIsAtomicCommitReady(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void FdwXactMarkForeignServerAccessed(Relation rel, int flags, bool modified);
+extern bool check_foreign_twophase_commit(int *newval, void **extra,
+										  GucSource source);
+
+#endif   /* FDW_XACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000..4ea65b2
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,32 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _FDWXACT_LAUNCHER_H
+#define _FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherWakeupToRequest(void);
+extern void FdwXactLauncherWakeupToRetry(void);
+
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+
+extern bool IsFdwXactLauncher(void);
+
+extern void fdwxact_maybe_launch_resolver(bool ignore_error);
+
+
+#endif	/* _FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000..6b2a24f
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int foreign_xact_resolver_timeout;
+
+#endif		/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000..e92b5a1
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,52 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDW_XACT_INSERT	0x00
+#define XLOG_FDW_XACT_REMOVE	0x10
+
+/* Same as GIDSIZE */
+#define FDW_XACT_MAX_ID_LEN 200
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdw_xact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+} xl_fdw_xact_remove;
+
+extern void fdw_xact_redo(XLogReaderState *record);
+extern void fdw_xact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdw_xact_identify(uint8 info);
+
+#endif	/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000..36391d4
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,67 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/resovler_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _RESOLVER_INTERNAL_H
+#define _RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t	pid;	/* this resolver's PID, or 0 if not active */
+	Oid		dbid;	/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool	in_use;
+
+	/* Stats */
+	TimestampTz	last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t	mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	*latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/*
+	 * Foreign transaction resolution queues. Protected by FdwXactLock.
+	 */
+	SHM_QUEUE	FdwXactActiveQueue;
+	SHM_QUEUE	FdwXactRetryQueue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch		*launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif	/* _RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 0bbe9879..c15dff7 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDW_XACT_ID, "Foreign Transactions", fdw_xact_redo, fdw_xact_desc, fdw_xact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 0e932da..b199c88 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 				TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 2c1b2d8..63c833d 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -105,6 +105,13 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE				(1U << 3)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 30610b3..795e85a 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -227,6 +227,7 @@ typedef struct xl_parameter_change
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index 773d9e6..3d5333a 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -178,6 +178,7 @@ typedef struct ControlFileData
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 4d7fe1b..599ce8c 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5032,6 +5032,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '6053', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_fdwxact_resolver', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_fdwxact_resolver' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5737,6 +5744,22 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '6050', descr => 'view foreign transactions',
+  proname => 'pg_prepared_fdw_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,text}',
+  proargmodes => '{o,o,o,o,o,o}',
+  proargnames => '{dbid,transaction,serverid,userid,status,identifier}',
+  prosrc => 'pg_prepared_fdw_xacts' },
+{ oid => '6051', descr => 'remove foreign transaction',
+  proname => 'pg_remove_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_remove_fdw_xact' },
+{ oid => '6052', descr => 'resolve foreign transaction',
+  proname => 'pg_resolve_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_resolve_fdw_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index c14eb54..92d47bb 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/relation.h"
@@ -168,6 +169,14 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef bool (*PrepareForeignTransaction_function) (FdwXactState *state);
+typedef bool (*CommitForeignTransaction_function) (FdwXactState *state);
+typedef bool (*RollbackForeignTransaction_function) (FdwXactState *state);
+typedef bool (*ResolveForeignTransaction_function) (FdwXactState *state,
+													bool is_commit);
+typedef bool (*IsTwoPhaseCommitEnabled_function) (Oid serverid);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -235,6 +244,14 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for distributed transactions */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	ResolveForeignTransaction_function ResolveForeignTransaction;
+	IsTwoPhaseCommitEnabled_function IsTwoPhaseCommitEnabled;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
@@ -247,7 +264,6 @@ typedef struct FdwRoutine
 	ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild;
 } FdwRoutine;
 
-
 /* Functions in foreign/foreign.c */
 extern FdwRoutine *GetFdwRoutine(Oid fdwhandler);
 extern Oid	GetForeignServerIdByRelId(Oid relid);
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 3ca12e6..d030368 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -68,10 +68,10 @@ typedef struct ForeignTable
 	List	   *options;		/* ftoptions as DefElem list */
 } ForeignTable;
 
-
 extern ForeignServer *GetForeignServer(Oid serverid);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperByName(const char *name,
 							bool missing_ok);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index f1c10d1..05feb0a 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -759,6 +759,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDW_XACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -833,7 +835,8 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDW_XACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -913,6 +916,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_READ,
+	WAIT_EVENT_FDW_XACT_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index cb613c8..45880b2 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -153,6 +153,16 @@ struct PGPROC
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction
+								 * resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+
+	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
 	 * their lock.
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index 75bab29..25d6a2f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDW_XACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -124,4 +126,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 								TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 668d9ef..81560bd 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -94,6 +94,8 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 735dd37..fdd6ded 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1413,6 +1413,13 @@ pg_policies| SELECT n.nspname AS schemaname,
    FROM ((pg_policy pol
      JOIN pg_class c ON ((c.oid = pol.polrelid)))
      LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace)));
+pg_prepared_fdw_xacts| SELECT f.dbid,
+    f.transaction,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.identifier
+   FROM pg_prepared_fdw_xacts() f(dbid, transaction, serverid, userid, status, identifier);
 pg_prepared_statements| SELECT p.name,
     p.statement,
     p.prepare_time,
@@ -1821,6 +1828,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_fdwxact_resolvers| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_fdwxact_resolver() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_progress_vacuum| SELECT s.pid,
     s.datid,
     d.datname,
-- 
2.10.5

From 67010b0b7965045df805cb3e96bce110d5d88ddf Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Thu, 8 Feb 2018 11:26:46 +0900
Subject: [PATCH v22 1/4] Keep track of writing on non-temporary relation.

---
 src/backend/access/heap/heapam.c | 12 ++++++++++++
 src/include/access/xact.h        |  5 +++++
 2 files changed, 17 insertions(+)

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index fb63471..c2db19b 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2629,6 +2629,10 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 		heap_freetuple(heaptup);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleGetOid(tup);
 }
 
@@ -3453,6 +3457,10 @@ l1:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	return HeapTupleMayBeUpdated;
 }
 
@@ -4403,6 +4411,10 @@ l2:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(relation))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	bms_free(hot_attrs);
 	bms_free(proj_idx_attrs);
 	bms_free(key_attrs);
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 689c57c..2c1b2d8 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -98,6 +98,11 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
-- 
2.10.5

From f86094413a1195f5a7c24d3ba834e1b832592d76 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:46:01 +0900
Subject: [PATCH v22 3/4] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/Makefile                  |   7 +-
 contrib/postgres_fdw/connection.c              | 609 ++++++++++++++++---------
 contrib/postgres_fdw/expected/postgres_fdw.out | 344 +++++++++++++-
 contrib/postgres_fdw/fdwxact.conf              |   3 +
 contrib/postgres_fdw/option.c                  |   5 +-
 contrib/postgres_fdw/postgres_fdw.c            |  58 ++-
 contrib/postgres_fdw/postgres_fdw.h            |  11 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql      | 159 +++++++
 doc/src/sgml/postgres-fdw.sgml                 |  37 ++
 9 files changed, 989 insertions(+), 244 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index 85394b4..5198f40 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -10,7 +10,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -23,3 +23,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index fe4893a..494491c 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -14,9 +14,12 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
-#include "catalog/pg_user_mapping.h"
 #include "access/xact.h"
+#include "catalog/pg_user_mapping.h"
+#include "commands/defrem.h"
+#include "foreign/fdwapi.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -45,7 +48,7 @@
  */
 typedef Oid ConnCacheKey;
 
-typedef struct ConnCacheEntry
+struct ConnCacheEntry
 {
 	ConnCacheKey key;			/* hash key (must be first) */
 	PGconn	   *conn;			/* connection to foreign server, or NULL */
@@ -56,9 +59,21 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
-} ConnCacheEntry;
+};
+
+/*
+ * Foreign transaction state using postgres_fdw.
+ */
+typedef struct PgFdwXactState
+{
+	Oid		serverid;
+	Oid		userid;
+	Oid		umid;
+	ConnCacheEntry	*conn;
+} PgFdwXactState;
 
 /*
  * Connection cache (initialized on first use)
@@ -69,17 +84,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 					   SubTransactionId mySubid,
 					   SubTransactionId parentSubid,
@@ -91,24 +102,20 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 						 bool ignore_errors);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 						 PGresult **result);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
-
-/*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
- */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +135,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,16 +142,11 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
 	entry = hash_search(ConnectionHash, &key, HASH_ENTER, &found);
+
 	if (!found)
 	{
 		/*
@@ -155,6 +156,17 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -182,6 +194,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +203,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +214,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,11 +230,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -414,7 +465,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -644,193 +695,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 }
 
 /*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow remote transactions that modified anything,
-					 * since it's not very reasonable to hold them open until
-					 * the prepared transaction is committed.  For the moment,
-					 * throw error unconditionally; later we might allow
-					 * read-only cases.  Note that the error will cause us to
-					 * come right back here with event == XACT_EVENT_ABORT, so
-					 * we'll clean up the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot prepare a transaction that modified remote tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
-/*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
 static void
@@ -846,10 +710,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -860,6 +720,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1193,3 +1057,302 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ *
+ * This function is called only at the pre-commit phase of the local transaction.
+ */
+bool
+postgresPrepareForeignTransaction(FdwXactState *state)
+{
+	PgFdwXactState *rstate;
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+	PGresult	*res;
+	StringInfo	command;
+
+	entry = GetConnectionCacheEntry(state->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	rstate = (PgFdwXactState *) palloc0(sizeof(PgFdwXactState));
+	rstate->serverid = state->serverid;
+	rstate->userid = state->userid;
+	rstate->umid = state->umid;
+	rstate->conn = entry;
+	state->fdw_state = (void *)rstate;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+	{
+		result = true;
+		elog(DEBUG1, "prepared foreign transaction on server %u with ID %s",
+			 state->serverid, state->fdwxact_id);
+	}
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/*
+ * Commit a transaction on foreign server.
+ *
+ * This function is called both at the pre-commit phase of the local transaction.
+ */
+bool
+postgresCommitForeignTransaction(FdwXactState *state)
+{
+	PgFdwXactState *rstate;
+	ConnCacheEntry *entry = NULL;
+	bool		result = false;
+	PGresult	*res;
+
+	entry = GetConnectionCacheEntry(state->umid);
+
+	if (!entry || !entry->conn || !entry->xact_got_connection)
+		return true;
+
+	rstate = (PgFdwXactState *) palloc0(sizeof(PgFdwXactState));
+	rstate->serverid = state->serverid;
+	rstate->userid = state->userid;
+	rstate->umid = state->umid;
+	rstate->conn = entry;
+	state->fdw_state = (void *)rstate;
+
+	/*
+	 * If abort cleanup previously failed for this connection,
+	 * we can't issue any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) == PGRES_COMMAND_OK)
+		result = true;
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/*
+ * Rollback a transaction on foreign server.
+ *
+ * This function is called each time when aborting.
+ */
+bool
+postgresRollbackForeignTransaction(FdwXactState *state)
+{
+	PgFdwXactState *rstate = (PgFdwXactState *) state->fdw_state;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (rstate)
+		entry = rstate->conn;
+	else
+		entry = GetConnectionCacheEntry(state->umid);
+
+	if (!entry || !entry->conn)
+		return true;
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return true;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return !abort_cleanup_failure;
+}
+
+/*
+ * Resolve a prepared transaction on foreign server.
+ *
+ * This function is called after committed locally by either a foreign transaction
+ * resolver or pg_resolve_fdw_xact.
+ */
+bool
+postgresResolveForeignTransaction(FdwXactState *state, bool is_commit)
+{
+	ConnCacheEntry *entry = NULL;
+	StringInfo	command;
+	bool result = true;
+	PGresult	*res;
+
+	entry = GetConnectionState(state->umid, false, false);
+
+	if (!entry->conn)
+		return false;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 state->fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * If we tried to COMMIT/ABORT a prepared transaction and the prepared
+		 * transaction was missing on the foreign server, it was probably
+		 * resolved by some other means. Anyway, it should be considered as resolved.
+		 */
+		result = (sqlstate == ERRCODE_UNDEFINED_OBJECT);
+
+		/*
+		 * The command failed, raise a warning to log the reason of failure.
+		 * We may not be in a transaction here, so raising error doesn't
+		 * help.
+		 */
+		pgfdw_report_error(WARNING, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction on server %u with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 state->serverid,
+		 state->fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return result;
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 21a2ef5..c63885d 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,15 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -185,16 +207,19 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                                      List of foreign tables
- Schema |   Table    |  Server   |                   FDW options                    | Description 
---------+------------+-----------+--------------------------------------------------+-------------
- public | ft1        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft2        | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
- public | ft4        | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
- public | ft5        | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft6        | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
- public | ft_pg_type | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
-(6 rows)
+                                         List of foreign tables
+ Schema |      Table       |  Server   |                   FDW options                    | Description 
+--------+------------------+-----------+--------------------------------------------------+-------------
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1')            | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')            | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')            | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')            | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type') | 
+(9 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8650,3 +8675,302 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+\det+
+                                                 List of foreign tables
+ Schema |      Table       |  Server   |                            FDW options                            | Description 
+--------+------------------+-----------+-------------------------------------------------------------------+-------------
+ public | fpagg_tab_p1     | loopback  | (table_name 'pagg_tab_p1')                                        | 
+ public | fpagg_tab_p2     | loopback  | (table_name 'pagg_tab_p2')                                        | 
+ public | fpagg_tab_p3     | loopback  | (table_name 'pagg_tab_p3')                                        | 
+ public | ft1              | loopback  | (schema_name 'S 1', table_name 'T 1')                             | 
+ public | ft2              | loopback  | (schema_name 'S 1', table_name 'T 1', use_remote_estimate 'true') | 
+ public | ft3              | loopback  | (table_name 'loct3', use_remote_estimate 'true')                  | 
+ public | ft4              | loopback  | (schema_name 'S 1', table_name 'T 3')                             | 
+ public | ft5              | loopback  | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft6              | loopback2 | (schema_name 'S 1', table_name 'T 4')                             | 
+ public | ft7_twophase     | loopback  | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft8_twophase     | loopback2 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft9_not_twophase | loopback3 | (schema_name 'S 1', table_name 'T 5')                             | 
+ public | ft_pg_type       | loopback  | (schema_name 'pg_catalog', table_name 'pg_type')                  | 
+ public | ftprt1_p1        | loopback  | (table_name 'fprt1_p1', use_remote_estimate 'true')               | 
+ public | ftprt1_p2        | loopback  | (table_name 'fprt1_p2')                                           | 
+ public | ftprt2_p1        | loopback  | (table_name 'fprt2_p1', use_remote_estimate 'true')               | 
+ public | ftprt2_p2        | loopback  | (table_name 'fprt2_p2', use_remote_estimate 'true')               | 
+ public | rem1             | loopback  | (table_name 'loc1')                                               | 
+ public | rem2             | loopback  | (table_name 'loc2')                                               | 
+(19 rows)
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+  srvname  
+-----------
+ loopback
+ loopback2
+ loopback3
+(3 rows)
+
+-- Enable atomic commit
+SET distributed_atomic_commit TO 'required';
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(5);
+INSERT INTO ft9_not_twophase VALUES(5);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+-- Check differences between configuration when a transaction mofieid
+-- data on both 2pc-capable and non-2pc-capable servers.
+-- When set to 'required' it fails.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+ERROR:  cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+-- When set to 'prefer', we can commit it
+SET distributed_atomic_commit TO 'prefer';
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+  7
+  7
+(6 rows)
+
+-- But cannot prepare the local transaction
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+PREPARE TRANSACTION 'gx1'; -- error
+ERROR:  can not prepare the transaction because some foreign servers involved in transaction can not prepare the transaction
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+  7
+  7
+(6 rows)
+
+-- When set to 'disabled', we can commit it
+SET distributed_atomic_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft8_twophase VALUES(8);
+INSERT INTO ft9_not_twophase VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+  7
+  7
+  8
+  8
+(8 rows)
+
+-- Similary, but cannot prepare the local transaction
+BEGIN;
+INSERT INTO ft8_twophase VALUES(8);
+INSERT INTO ft9_not_twophase VALUES(8);
+PREPARE TRANSACTION 'gx1'; -- error
+ERROR:  cannot PREPARE a distributed transaction when distributed_atomic_commit is 'disabled'
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+  7
+  7
+  8
+  8
+(8 rows)
+
+SET distributed_atomic_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+  7
+  7
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_twophase;
+ c1 
+----
+  1
+  2
+  2
+  3
+  7
+  7
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_prepared_fdw_xacts;
+ count 
+-------
+     0
+(1 row)
+
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000..3fdbf93
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/option.c b/contrib/postgres_fdw/option.c
index 6854f1b..404b318 100644
--- a/contrib/postgres_fdw/option.c
+++ b/contrib/postgres_fdw/option.c
@@ -108,7 +108,8 @@ postgres_fdw_validator(PG_FUNCTION_ARGS)
 		 * Validate option value, when we can do so without any context.
 		 */
 		if (strcmp(def->defname, "use_remote_estimate") == 0 ||
-			strcmp(def->defname, "updatable") == 0)
+			strcmp(def->defname, "updatable") == 0 ||
+			strcmp(def->defname, "two_phase_commit") == 0)
 		{
 			/* these accept only boolean values */
 			(void) defGetBoolean(def);
@@ -177,6 +178,8 @@ InitPgFdwOptions(void)
 		/* fetch_size is available on both server and table */
 		{"fetch_size", ForeignServerRelationId, false},
 		{"fetch_size", ForeignTableRelationId, false},
+		/* two-phase commit support */
+		{"two_phase_commit", ForeignServerRelationId, false},
 		{NULL, InvalidOid, false}
 	};
 
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index fd20aa9..5214627 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "catalog/pg_class.h"
@@ -359,6 +360,7 @@ static void postgresGetForeignUpperPaths(PlannerInfo *root,
 							 RelOptInfo *input_rel,
 							 RelOptInfo *output_rel,
 							 void *extra);
+static bool postgresIsTwoPhaseCommitEnabled(Oid serverid);
 
 /*
  * Helper functions
@@ -452,7 +454,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 				  const PgFdwRelationInfo *fpinfo_o,
 				  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -506,10 +507,28 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+	routine->ResolveForeignTransaction = postgresResolveForeignTransaction;
+	routine->IsTwoPhaseCommitEnabled = postgresIsTwoPhaseCommitEnabled;
+
 	PG_RETURN_POINTER(routine);
 }
 
 /*
+ * postgresIsTwoPhaseCommitEnabled
+ */
+static bool
+postgresIsTwoPhaseCommitEnabled(Oid serverid)
+{
+	ForeignServer	*server = GetForeignServer(serverid);
+
+	return server_uses_twophase_commit(server);
+}
+
+/*
  * postgresGetForeignRelSize
  *		Estimate # of rows and width of the result of the scan
  *
@@ -1356,7 +1375,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2411,7 +2430,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2704,7 +2723,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								&retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3321,7 +3340,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4108,7 +4127,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4198,7 +4217,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4421,7 +4440,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
@@ -5803,3 +5822,26 @@ find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel)
 	/* We didn't find any suitable equivalence class expression */
 	return NULL;
 }
+
+/*
+ * server_uses_twophase_commit
+ * Returns true if the foreign server is configured to support 2PC.
+ */
+bool
+server_uses_twophase_commit(ForeignServer *server)
+{
+	ListCell		*lc;
+
+	/* Check the options for two phase compliance */
+	foreach(lc, server->options)
+	{
+		DefElem    *d = (DefElem *) lfirst(lc);
+
+		if (strcmp(d->defname, "two_phase_commit") == 0)
+		{
+			return defGetBoolean(d);
+		}
+	}
+	/* By default a server is not 2PC compliant */
+	return false;
+}
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index 70b538e..3526923 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/relation.h"
@@ -110,12 +111,14 @@ typedef struct PgFdwRelationInfo
 	int			relation_index;
 } PgFdwRelationInfo;
 
+typedef struct ConnCacheEntry ConnCacheEntry;
+
 /* in postgres_fdw.c */
 extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -123,6 +126,11 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 				   bool clear, const char *sql);
+extern bool postgresPrepareForeignTransaction(FdwXactState *state);
+extern bool postgresCommitForeignTransaction(FdwXactState *state);
+extern bool postgresRollbackForeignTransaction(FdwXactState *state);
+extern bool postgresResolveForeignTransaction(FdwXactState *state,
+											  bool is_commit);
 
 /* in option.c */
 extern int ExtractConnectionOptions(List *defelems,
@@ -181,6 +189,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 						List *remote_conds, List *pathkeys, bool is_subquery,
 						List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 88c4cb4..49b4f4c 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,19 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_twophase (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_twophase (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft9_not_twophase (
+       c1 int NOT NULL
+) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- A table with oids. CREATE FOREIGN TABLE doesn't support the
 -- WITH OIDS option, but ALTER does.
 CREATE FOREIGN TABLE ft_pg_type (
@@ -2354,3 +2381,135 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+
+-- ===================================================================
+-- test atomic commit across foreign servers
+-- ===================================================================
+
+ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on');
+ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'off');
+
+\det+
+
+-- Check two_phase_commit setting
+SELECT srvname FROM pg_foreign_server WHERE 'two_phase_commit=on' = ANY(srvoptions) or 'two_phase_commit=off' = ANY(srvoptions);
+
+-- Enable atomic commit
+SET distributed_atomic_commit TO 'required';
+
+-- Modify one 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+COMMIT;
+SELECT * FROM ft7_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+
+-- Modify two 2PC-capable servers then commit and rollback.
+-- This requires to use 2PC when commit.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+COMMIT;
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(2);
+INSERT INTO ft8_twophase VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_twophase;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error
+BEGIN;
+INSERT INTO ft7_twophase VALUES(4);
+INSERT INTO ft8_twophase VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+
+-- Rollback foreign transaction that involves both 2PC-capable
+-- and 2PC-non-capable foreign servers.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(5);
+INSERT INTO ft9_not_twophase VALUES(5);
+ROLLBACK;
+SELECT * FROM ft8_twophase;
+
+-- Check differences between configuration when a transaction mofieid
+-- data on both 2pc-capable and non-2pc-capable servers.
+
+-- When set to 'required' it fails.
+BEGIN;
+INSERT INTO ft8_twophase VALUES(6);
+INSERT INTO ft9_not_twophase VALUES(6);
+COMMIT; -- error
+SELECT * FROM ft8_twophase;
+
+-- When set to 'prefer', we can commit it
+SET distributed_atomic_commit TO 'prefer';
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+
+-- But cannot prepare the local transaction
+BEGIN;
+INSERT INTO ft8_twophase VALUES(7);
+INSERT INTO ft9_not_twophase VALUES(7);
+PREPARE TRANSACTION 'gx1'; -- error
+SELECT * FROM ft8_twophase;
+
+-- When set to 'disabled', we can commit it
+SET distributed_atomic_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft8_twophase VALUES(8);
+INSERT INTO ft9_not_twophase VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft8_twophase;
+
+-- Similary, but cannot prepare the local transaction
+BEGIN;
+INSERT INTO ft8_twophase VALUES(8);
+INSERT INTO ft9_not_twophase VALUES(8);
+PREPARE TRANSACTION 'gx1'; -- error
+SELECT * FROM ft8_twophase;
+
+SET distributed_atomic_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_twophase;
+
+BEGIN;
+INSERT INTO ft7_twophase VALUES(9);
+INSERT INTO ft8_twophase VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_twophase;
+
+-- No entry remained
+SELECT count(*) FROM pg_prepared_fdw_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 54b5e98..f4a9ff5 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -436,6 +436,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
-- 
2.10.5

From 11d87cacb90678b68a57363611060081aebbc0ae Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:48:08 +0900
Subject: [PATCH v22 4/4] Add regression tests for atomic commit.

---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index daf79a0..71c8b9d 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000..640b206
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+distributed_atomic_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port', two_phase_commit 'on');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port', two_phase_commit 'on');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_prepared_fdw_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 3248603..c1cd8ae 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2288,9 +2288,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2305,7 +2308,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.10.5

From d4eb4b8bdb36ec928bb14b081f9067f223afa791 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:44:42 +0900
Subject: [PATCH v22 2/4] Support atomic commit among multiple foreign servers.

---
 doc/src/sgml/catalogs.sgml                    |   97 +
 doc/src/sgml/config.sgml                      |  143 +-
 doc/src/sgml/distributed-transaction.sgml     |  157 ++
 doc/src/sgml/fdwhandler.sgml                  |  203 ++
 doc/src/sgml/filelist.sgml                    |    1 +
 doc/src/sgml/func.sgml                        |   51 +
 doc/src/sgml/monitoring.sgml                  |   60 +
 doc/src/sgml/postgres.sgml                    |    1 +
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/fdwxact.c          | 2678 +++++++++++++++++++++++++
 src/backend/access/fdwxact/fdwxact_launcher.c |  641 ++++++
 src/backend/access/fdwxact/fdwxact_resolver.c |  331 +++
 src/backend/access/heap/heapam.c              |   12 -
 src/backend/access/rmgrdesc/Makefile          |    8 +-
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   65 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/Makefile           |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   32 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/copy.c                   |    7 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    8 +
 src/backend/executor/nodeForeignscan.c        |   25 +
 src/backend/executor/nodeModifyTable.c        |   24 +
 src/backend/foreign/foreign.c                 |   43 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   18 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    2 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   80 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  149 ++
 src/include/access/fdwxact_launcher.h         |   32 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   52 +
 src/include/access/resolver_internal.h        |   67 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   23 +
 src/include/foreign/fdwapi.h                  |   18 +-
 src/include/foreign/foreign.h                 |    2 +-
 src/include/pgstat.h                          |    8 +-
 src/include/storage/proc.h                    |   10 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    2 +
 src/test/regress/expected/rules.out           |   12 +
 63 files changed, 5324 insertions(+), 40 deletions(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100755 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/fdwxact_launcher.c
 create mode 100644 src/backend/access/fdwxact/fdwxact_resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 8b7f169..f2f0571 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9597,6 +9597,103 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-prepared-fdw-xacts">
+  <title><structname>pg_prepared_fdw_xacts</structname></title>
+
+  <indexterm zone="view-pg-prepared-fdw-xacts">
+   <primary>pg_prepared_fdw_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_prepared_fdw_xacts</structname> displays
+   information about foreign transactions that are currently prepared on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="fdw-transaction-managements"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_prepared_xacts</structname> contains one row per prepared
+   foreign transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_prepared_fdw_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>transaction</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Transaction id that this foreign transaction associates with
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server that this foreign server is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction: <literal>prepared</literal>, <literal>committing</literal>, <literal>aborting</literal> or <literal>unknown</literal>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 0f8f2ef..4fffb76 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -3611,7 +3611,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -7827,6 +7826,148 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-distributed-atomic-commit" xreflabel="distributed_atomic_commit">
+       <term><varname>distributed_atomic_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>distributed_atomic_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign transaction
+         to be resolved before the command returns a "success" indication to the client.
+         Valid values are <literal>required</literal>, <literal>prefer</literal> and
+         <literal>disabled</literal>. The default setting is <literal>disabled</literal>.
+         When <literal>disabled</literal>, there can be risk of database consistency among
+         distributed transaction if some foreign server crashes during committing the
+         distributed transaction. When set to <literal>required</literal> the distributed
+         transaction requires that all written servers can use two-phase commit protocol.
+         That is, the transaction fails if any of servers returns <literal>false</literal>
+         from <function>IsTwoPhaseCommitEnabled</function> or does not support transaction
+         management callback routines(described in
+         <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction requires
+         two-phase commit protocol where available but without failing when it is not
+         available.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero value to
+         set this parameter either <literal>required</literal> or <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one transaction
+         is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism.  You should set this value to
+         zero only if you set <varname>max_foreign_transaction_resolvers</varname> as
+         much as databases you have. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000..deb8a60
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,157 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction Management</title>
+
+ <para>
+  This chapter explains what distributed transaction management is, and how it can be configured
+  in PostgreSQL.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit is an operation that applies a set of changes as a single operation
+   globally. <productname>PostgreSQL</productname> provides a way to perform a transaction
+   with foreign resources using <literal>Foreign Data Wrapper</literal>. Using the
+   <productname>PostgreSQL</productname>'s atomic commit ensures that all changes
+   on foreign servers end in either commit or rollback using the transaction callback
+   routines (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs Two-phase commit protocol, which is a
+    type of atomic commitment protocol (ACP). Using Two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed transaction manager
+    prepares all transaction on the foreign servers if two-phase commit is required.
+    Two-phase commit is required only if the transaction modifies data on two or more
+    servers including the local server itself and user requests it by
+    <xref linkend="guc-distributed-atomic-commit"/>. If all preparations on foreign servers
+    got successful go to the next step. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes over rollback, then rollback all transactions
+    on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the transaction
+    locally. Any failure happens in this step <productname>PostgreSQL</productname> changes
+    over rollback, then rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign Transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Foreign Transaction Resolution</title>
+
+   <para>
+    Foreign transaction resolutions are performed by foreign transaction resolver process.
+    They commit all prepared transaction on foreign servers if the coordinator received
+    an agreement message from all foreign server during the first step. On the other hand,
+    if any foreign server failed to prepare the transaction, it rollbacks all prepared
+    transactions.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions on one
+    database of the coordinator side. On failure during resolution, they retries to
+    resolve after <varname>foreign_transaction_resolution_interval</varname>.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>In-doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit or rollback
+    using two-phase commit protocol. However, if the second phase fails for whatever reason
+    the transaction becomes in-doubt. The transactions becomes in-doubt in the following
+    situations:
+
+   <itemizedlist>
+    <listitem>
+     <para>
+      A local <productname>PostgreSQL</productname> server crashes during atomic commit
+      operation.
+     </para>
+    </listitem>
+
+    <listitem>
+     <para>
+      A local <productname>PostgreSQL</productname> server got a cancellation by user during
+      atomic commit.
+     </para>
+    </listitem>
+   </itemizedlist>
+
+   In-doubt transactions are automatically handled by foreign transaction resolver process
+   when there is no online transaction requesting resolutions.
+   <function>pg_resolve_fdw_xact</function> provides a way to resolve transactions on foreign
+   servers manually that participated the distributed transaction manually.
+   </para>
+
+   <para>
+    The atomic commit operation is crash-safe. The being processed foreign transactions at
+    crash are processed by a foreign transaction resolvers as an in-doubt transaction
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-fdwxact-resolver-view"><literal>pg_stat_fdwxact_resolver</literal></link>
+    view. This view contains one row for every foreign Transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 4ce88dd..90cc415 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1390,6 +1390,118 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     If an FDW wishes to support <firstterm>atomic commit</firstterm>
+     (as described in <xref linkend="fdw-transaction-managements"/>), it must call the
+     registrasaction function <function>FdwXactRegisterForeignTransaction</function>
+     and provide the following callback functions:
+    </para>
+
+    <para>
+<programlisting>
+bool
+PrepareForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if atomic commit is required.
+    Returning <literal>true</literal> means that preparing the foreign
+    transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Commit the not-prepared transaction on the foreign server.
+    This function is called at the pre-commit phase of local
+    transaction if atomic commit is not required. The atomic
+    commit is not required either when we modified data on
+    only one server including the local server or when userdoesn't
+    request atomic commit by <xref linkend="guc-distributed-atomic-commit"/>.
+    Returning <literal>true</literal> means that commit the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactResolveState *state);
+</programlisting>
+    Rollback a not-prepared transaction on the foreign server.
+    This function is called at the end of local transaction after
+    rollbacked locally either when user requested rollback or when
+    any error occurs during the transaction. This function could
+    be called recursively if any error occurs during rollback the
+    foreign transaction for whatever reason. You need to track
+    recursion and prevent this function from being called infinitely.
+    Returning <literal>true</literal> means that rollback the
+    foreign transaction got successful.
+    </para>
+    <para>
+<programlisting>
+bool
+ResolvePreparedForeignTransaction(FdwXactResolveState *state,
+                                  bool is_commit);
+</programlisting>
+    Commit or rollback the prepared transaction on the foreign server.
+    When <varname>is_commit</varname> is true, it indicates that the foreign
+    transaction should be committed. Otherwise the foreign transaction should
+    be aborted.
+    This function normally is called by the foreign transaction resolver
+    process but can also be called by <function>pg_resovle_fdw_xacts</function>
+    function. In the resolver process, this function is called either
+    when a backend requests the resolver process to resolve a distributed
+    transaction after prepared, or when a database has dangling
+    transactions. Returning <literal>true</literal> means that resolving
+    the foreign transaction got successful.
+    In abort case, please note that the prepared transaction identified
+    by <varname>state->fdwxact_id</varname> might not exist on the foreign
+    server. If you failed to resolve the foreign transaction due to undefined
+    object error (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) you should
+   regards it as success and return <literal>true</literal>.
+    </para>
+    <para>
+<programlisting>
+bool
+IsTwoPhaseCommitEnabled(Oid serverid);
+</programlisting>
+    Return <literal>true</literal> if the foreign server identified by
+    <literal>serverid</literal> is capable of two-phase commit protocol.
+    This function is called at commit time once.
+    Return <literal>false</literal> indicates that the current transaction
+    cannot use atomic commit even if atomic commit is requested by user.
+    </para>
+
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1835,4 +1947,95 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+    <title>Transaction managements for Foreign Data Wrappers</title>
+
+    <para>
+     <productname>PostgreSQL</productname> foreign transaction manager
+     allows FDWs to read and write data on foreign server within a transaction while
+     maintaining atomicity of the foreign data (aka atomic commit). Using
+     atomic commit, it guarantees that a distributed transaction is committed
+     or rollbacked on all participants foreign
+     server.  To achieve atomic commit, <productname>PostgreSQL</productname>
+     employees two-phase commit protocol, which is a type of atomic commitment
+     protocol. Every FDW that wish to support atomic commit
+     is required to support the transaction management callback routines:
+     <function>PrepareForeignTransaction</function>,
+     <function>CommitForeignTransaction</function>,
+     <function>RollbackForeignTransaction</function>,
+     <function>ResolveForeignTransaction</function>,
+     <function>IsTwoPhaseCommitEnabled</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+     Tranasction of foreign server that supports these callback routines is
+     managed by <productname>PostgreSQL</productname>'s distributed  transaction
+     manager. Each transaction management callbacks are called at appropriate time.
+    </para>
+
+    <para>
+     The information in <literal>FdwXactState</literal> can be used to identify
+     foreign servers. <literal>state-&gt;fdw_state</literal> is a <type>void</type>
+     pointer that is available for FDW transaction functions to store Information
+     relevant to the particular foreign server.  It is useful for passing
+     information forward from <function>PrepareForeignTransaction</function> and/or
+     <function>CommitTransaciton</function> to
+     <function>RollbackForeignTransaction</function>, there by avoiding recalculation.
+     Note that since <function>ResolveForeignTransaction</function> is called
+     idependently from these callback routines, the information is not passed to
+     <function>ResolverForeignTransaction</function>.
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling <function>PrepareForeignTransaction</function>
+     if two-phase commit protocol is required. Two-phase commit is required only if
+     the transaction modified data on more than one servers including the local
+     server itself and user requests atomic commit. <productname>PostgreSQL</productname>
+     can commit locally and go to the next step if and only if all preparing foreign
+     transactions got successful. If two-phase commit is not required, the foreign
+     transaction manager commits each transaction calling
+     <function>CommitForeignTransaction</function> and then commit locally.
+     If any failure happens or user requests to cancel during the pre-commit phase
+     the distributed Transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function> for not-prepared foreign
+     servers, and then rollback locally. The prepared foreign servers are rollbacked
+     by a foreign transaction resolver process.
+    </para>
+
+    <para>
+     Once committed locally, the distributed transaction must be committed. The
+     prepared foreign transaction will be committed by foreign transaction resolver
+     process.
+    </para>
+
+    <para>
+     When two-phase commit is required, after committed locally, the transaction
+     commit will wait for all prepared foreign transaction to be committed before
+     completetion. One foreign transaction resolver process is responsible for
+     foreign transaction resolution on a database.
+     <function>ResolverForeignTransaction</function> is called by the foreign
+     transaction resolver process when resolution.
+     <function>ResolveForeignTransaction</function> is also be called
+     when user executes <function>pg_resovle_fdw_xact</function> function.
+    </para>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 48ac14a..38d6fcb 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -49,6 +49,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index edeb3fd..d609324 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -20905,6 +20905,57 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-fdw-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_fdw_xacts</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_fdw_xacts</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function search for foreign transaction
+        matching the arguments and resolves then. This function won't resolve
+        a foreign transaction which is in progress, or one that is locked by some
+        other backend.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_fdw_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_fdw_xact</function>
+        except it remove foreign transaction entry without resolving.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 7aada14..53f9e72 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -332,6 +332,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_fdw_xact_resolver</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-fdwxact-resolver-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1198,6 +1206,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
+        <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
         </row>
@@ -1413,6 +1433,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
+        <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
          <entry>Waiting during base backup when throttling activity.</entry>
@@ -2222,6 +2246,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-fdwxact-resolver-view" xreflabel="pg_stat_fdw_xact_resolver">
+   <title><structname>pg_stat_fdw_xact_resolver</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index 0070603..c10e21f 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -164,6 +164,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index bd93a6a..4a1ebdc 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  tablesample transam
+			  tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000..9ddbb14
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o fdwxact_resolver.o fdwxact_launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100755
index 0000000..109d6a7
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2678 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL distributed transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers automically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * When a foreign data wrapper starts transaction on a foreign server that
+ * is capable of two-phase commit protocol, foreign data wrappers registers
+ * the foreign transaction using function FdwXactRegisterForeignTransaction()
+ * in order to participate to a group for atomic commit. Participants are
+ * identified by oid of foreign server and user. When the foreign transaction
+ * begins to modify data the executor marks it as modified using
+ * FdwXactMarkForeignTransactionModified().
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. After committing or rolling back locally, we
+ * notify the resolver process and tell it to commit or roll back those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail when an ERROR
+ * after already committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * waiters each time we receive a request. We have two queues: the active
+ * queue and the retry queue. The backend is inserted to the active queue at
+ * first, and then it is moved to the retry queue by the resolver process if
+ * the resolution fails. The backends in the retry queue are processed at
+ * interval of foreign_transaction_resolution_retry_interval.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or more
+ * servers including itself. In other case, all foreign transactions are
+ * committed during pre-commit.
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. dangling
+ * transaction). Dangling transactions are processed by the resolve process
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * 	* On PREPARE redo we add the foreign transaction to FdwXactCtl->fdw_xacts.
+ *	  We set fdw_xact->inredo to true for such entries.
+ *	* On Checkpoint redo, we iterate through FdwXactCtl->fdw_xacts entries that
+ *	  have set fdw_xact->inredo true and are behind the redo_horizon. We save
+ *    them to disk and then set fdw_xact->ondisk to true.
+ *	* On COMMIT and ABORT we delete the entry from FdwXactCtl->fdw_xacts.
+ *	  If fdw_xact->ondisk is true, we delete the corresponding file from
+ *	  the disk as well.
+ *  * RecoverFdwXacts loads all foreign transaction entries from disk into
+ *    memory at server startup.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Is atomic commit requested by user? */
+#define IsAtomicCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+#define IsAtomicCommitRequested() \
+	(IsAtomicCommitEnabled() && \
+	 (distributed_atomic_commit > DISTRIBUTED_ATOMIC_COMMIT_DISABLED))
+
+/* Structure to bundle the foreign transaction participant */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in global entry. NULL if
+	 * this foreign transaction is registered but not inserted
+	 * yet.
+	 */
+	FdwXact		fdw_xact;
+	char		*fdw_xact_id;
+
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	bool		modified;					/* true if modified the data on server */
+	bool		twophase_commit_enabled;	/* true if the server can execute
+											 * two-phase commit protocol */
+	void			*fdw_state;				/* fdw-private state */
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact;
+	CommitForeignTransaction_function	commit_foreign_xact;
+	RollbackForeignTransaction_function	rollback_foreign_xact;
+	GetPrepareId_function				get_prepareid;
+	IsTwoPhaseCommitEnabled_function	is_twophase_commit_enabled;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit.
+ * This list has only foreign servers that support atomic commit FDW
+ * API regardless of their configuration.
+ */
+static List *FdwXactAtomicCommitParticipants = NIL;
+static bool FdwXactAtomicCommitReady = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDW_XACTS_DIR "pg_fdw_xact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDW_XACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDW_XACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactCommitForeignTransaction(FdwXactParticipant *fdw_part);
+static bool FdwXactResolveForeignTransaction(FdwXactState *state, FdwXact fdwxact,
+											 int elevel);
+static void FdwXactComputeRequiredXmin(void);
+static bool FdwXactAtomicCommitRequired(void);
+static void FdwXactQueueInsert(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid, bool give_warnings);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							  bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid, Oid serverid,
+								  Oid userid, XLogRecPtr insert_start_lsn,
+								  bool give_warnings);
+static void register_fdw_xact(Oid serverid, Oid userid, bool modified);
+static List *get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						   bool need_lock);
+static FdwXact get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+								bool need_lock);
+static FdwXact get_all_fdw_xacts(int *length);
+static FdwXact insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+							   Oid umid, char *fdw_xact_id);
+static char *generate_fdw_xact_identifier(TransactionId xid, Oid serverid, Oid userid);
+static void remove_fdw_xact(FdwXact fdw_xact);
+static FdwXactState *create_fdw_xact_state(void);
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+int distributed_atomic_commit = DISTRIBUTED_ATOMIC_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+/*
+ * Remember accessed foreign server. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation		rel;
+	Oid				serverid;
+	Oid				userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdw_xact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdw_xact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign server identified by given server id must support atomic
+ * commit APIs. Registered foreign transaction are managed by foreign
+ * transaction manager until the end of the transaction.
+ */
+static void
+register_fdw_xact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant	*fdw_part;
+	ForeignServer 		*foreign_server;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	MemoryContext		old_ctx;
+	FdwRoutine			*routine;
+	ListCell	   		*lc;
+
+	/*
+	 * Participants information is needed at the end of a transaction, where
+	 * system cache are not available. Save it in TopTransactionContext
+	 * beforehand so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * If the being modified foreign server doesn't have the atomic commit API
+	 * we don't manage the foreign transaction in the distributed transaction
+	 * manager.
+	 */
+	if (routine->IsTwoPhaseCommitEnabled == NULL)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		pfree(routine);
+		return;
+	}
+
+	foreach(lc, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant	*fp = (FdwXactParticipant *) lfirst(lc);
+
+		if (fp->server->serverid == serverid &&
+			fp->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fp->modified |= modified;
+			pfree(routine);
+			return;
+		}
+	}
+
+	foreign_server = GetForeignServer(serverid);
+	fdw = GetForeignDataWrapper(foreign_server->fdwid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+	/* Make sure that the FDW has transaction handlers */
+	if (!routine->PrepareForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function provided for preparing foreign transaction for FDW %s",
+						fdw->fdwname)));
+	if (!routine->CommitForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to commit a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+	if (!routine->RollbackForeignTransaction)
+		ereport(ERROR,
+				(errmsg("no function to rollback a foreign transaction provided for FDW %s",
+						fdw->fdwname)));
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdw_xact_id = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdw_xact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->twophase_commit_enabled = true; /* by default, will be changed at pre-commit phase */
+	fdw_part->fdw_state = NULL;
+	fdw_part->prepare_foreign_xact = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact = routine->RollbackForeignTransaction;
+	fdw_part->is_twophase_commit_enabled = routine->IsTwoPhaseCommitEnabled;
+	fdw_part->get_prepareid = routine->GetPrepareId;
+
+	/* Add this foreign transaction to the participants list */
+	FdwXactAtomicCommitParticipants = lappend(FdwXactAtomicCommitParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/*
+ * FdwXactShmemSize
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdw_xacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * FdwXactShmemInit
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdw_xacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->freeFdwXacts = NULL;
+		FdwXactCtl->numFdwXacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdw_xacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdw_xacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdw_xacts[cnt].status = FDW_XACT_INITIAL;
+			fdw_xacts[cnt].fxact_free_next = FdwXactCtl->freeFdwXacts;
+			FdwXactCtl->freeFdwXacts = &fdw_xacts[cnt];
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * PreCommit_FdwXacts
+ *
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_atomic_commit;
+	ListCell	*lc;
+	ListCell	*next;
+	ListCell	*prev = NULL;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	need_atomic_commit = FdwXactAtomicCommitRequired();
+
+	/*
+	 * If 'require' case, we require all modified server have to be capable of
+	 * two-phase commit protocol.
+	 */
+	if (need_atomic_commit &&
+		distributed_atomic_commit == DISTRIBUTED_ATOMIC_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	/*
+	 * Commit transactions on foreign servers.
+	 *
+	 * Committed transactions are removed from FdwXactAtomicCommitParticipants
+	 * so that the later preparation can process only servers that requires to be commit
+	 * using two-phase commit protocol.
+	 */
+	for (lc = list_head(FdwXactAtomicCommitParticipants); lc != NULL; lc = next)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool can_commit = false;
+
+		next = lnext(lc);
+
+		if (!need_atomic_commit || !fdw_part->modified)
+		{
+			/*
+			 * We can commit not-modified servers and when the atomic commit is not
+			 * required.
+			 */
+			can_commit = true;
+		}
+		else if (distributed_atomic_commit == DISTRIBUTED_ATOMIC_COMMIT_PREFER &&
+				 !fdw_part->twophase_commit_enabled)
+		{
+			/* Also in 'prefer' case, non-2pc-capable servers can be committed */
+			can_commit = true;
+		}
+
+		if (can_commit)
+		{
+			/* Commit the foreign transaction */
+			FdwXactCommitForeignTransaction(fdw_part);
+
+			/* Delete it from the participant list */
+			FdwXactAtomicCommitParticipants =
+				list_delete_cell(FdwXactAtomicCommitParticipants, lc, prev);
+
+			continue;
+		}
+
+		prev = lc;
+	}
+
+	/*
+	 * If only one participant of all participants is modified, we can commit it.
+	 * This can avoid to use two-phase commit for only one server in the 'prefer' case
+	 * where the transaction has one 2pc-capable modified server and some modified
+	 * servers.
+	 */
+	if (list_length(FdwXactAtomicCommitParticipants) == 1 &&
+		(MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) == 0)
+	{
+		Assert(distributed_atomic_commit == DISTRIBUTED_ATOMIC_COMMIT_PREFER);
+		FdwXactCommitForeignTransaction(linitial(FdwXactAtomicCommitParticipants));
+		list_free(FdwXactAtomicCommitParticipants);
+		return;
+	}
+
+	FdwXactPrepareForeignTransactions();
+	/* keep FdwXactparticipantsForAC until the end of transaction */
+}
+
+/*
+ * FdwXactPrepareForeignTransactions
+ *
+ * Prepare all foreign transaction participants.  This function creates a prepared
+ * participants chain each time when we prepared a foreign transaction. The prepared
+ * participants chain is used to access all participants of distributed transaction
+ * quickly. If any one of them fails to prepare, we change over aborts.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	FdwXactState *state;
+	ListCell   *lcell;
+	FdwXact		prev_fdwxact = NULL;
+	TransactionId txid;
+
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	state = create_fdw_xact_state();
+
+	/* Loop over the foreign connections */
+	txid = GetTopTransactionId();
+	foreach(lcell, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXact		fdwxact;
+
+		/* Generate an unique identifier */
+		if (fdw_part->get_prepareid)
+		{
+			char *id;
+			int fdwxact_id_len = 0;
+
+			id = fdw_part->get_prepareid(txid, fdw_part->server->serverid,
+										 fdw_part->usermapping->userid,
+										 &fdwxact_id_len);
+
+			if (!id)
+				ereport(ERROR,
+						(errcode(ERRCODE_UNDEFINED_OBJECT),
+						 (errmsg("foreign transaction identifier is not provided"))));
+
+			/* Check length of foreign transaction identifier */
+			id[fdwxact_id_len] = '\0';
+			if (fdwxact_id_len > NAMEDATALEN)
+				ereport(ERROR,
+						(errcode(ERRCODE_NAME_TOO_LONG),
+						 errmsg("foreign transaction identifer \"%s\" is too long",
+								id),
+						 errdetail("foreign transaction identifier must be less than %d characters.",
+								   NAMEDATALEN)));
+
+			fdw_part->fdw_xact_id = pstrdup(id);
+		}
+		else
+			fdw_part->fdw_xact_id = generate_fdw_xact_identifier(txid,
+																 fdw_part->server->serverid,
+																 fdw_part->usermapping->userid);
+
+		/*
+		 * Insert the foreign transaction entry. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(txid, fdw_part);
+
+		state->serverid = fdw_part->server->serverid;
+		state->userid = fdw_part->usermapping->userid;
+		state->umid = fdw_part->usermapping->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		/*
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal). During abort processing,
+		 * we might try to resolve a never-prepared transaction, and get an error.
+		 * This is fine as long as the FDW provides us unique prepared transaction
+		 * identifiers.
+		 */
+		if (!fdw_part->prepare_foreign_xact(state))
+		{
+			/* Failed to prepare, change over aborts */
+			ereport(ERROR,
+					(errmsg("could not prepare transaction on foreign server %s",
+							fdw_part->server->servername)));
+		}
+
+		/* Keep fdw_state until end of transaction */
+		fdw_part->fdw_state = state->fdw_state;
+
+		/* Preparation is success, update its status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdw_part->fdw_xact->status = FDW_XACT_PREPARED;
+		fdw_part->fdw_xact = fdwxact;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Create a prepared participants chain, which is link-ed FdwXact entries
+		 * involving with this transaction.
+		 */
+		if (prev_fdwxact)
+		{
+			/* Append others to the tail */
+			Assert(fdwxact->fxact_next == NULL);
+			prev_fdwxact->fxact_next = fdwxact;
+		}
+	}
+}
+
+/*
+ * Commit the given foreign transaction.
+ */
+void
+FdwXactCommitForeignTransaction(FdwXactParticipant *fdw_part)
+{
+	FdwXactState *state;
+
+	state = create_fdw_xact_state();
+	state->serverid = fdw_part->server->serverid;
+	state->userid = fdw_part->usermapping->userid;
+	state->umid = fdw_part->usermapping->umid;
+	fdw_part->fdw_state = (void *) state;
+
+	if (!fdw_part->commit_foreign_xact(state))
+		ereport(ERROR,
+				(errmsg("could not commit foreign transaction on server %s",
+						fdw_part->server->servername)));
+}
+
+/*
+ * FdwXactInsertFdwXactEntry
+ *
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdw_xact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fxact;
+	FdwXactOnDiskData	*fxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact = insert_fdw_xact(MyDatabaseId, xid, fdw_part->server->serverid,
+							fdw_part->usermapping->userid,
+							fdw_part->usermapping->umid, fdw_part->fdw_xact_id);
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->held_by = MyBackendId;
+	fdw_part->fdw_xact = fxact;
+	LWLockRelease(FdwXactLock);
+
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdw_xact_id);
+	data_len = data_len + strlen(fdw_part->fdw_xact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fxact_file_data->dbid = MyDatabaseId;
+	fxact_file_data->local_xid = xid;
+	fxact_file_data->serverid = fdw_part->server->serverid;
+	fxact_file_data->userid = fdw_part->usermapping->userid;
+	fxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fxact_file_data->fdw_xact_id, fdw_part->fdw_xact_id,
+		   strlen(fdw_part->fdw_xact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fxact_file_data, data_len);
+	fxact->insert_end_lsn = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_INSERT);
+	XLogFlush(fxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fxact->valid = true;
+	LWLockRelease(FdwXactLock);
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fxact_file_data);
+	return fxact;
+}
+
+/*
+ * insert_fdw_xact
+ *
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				Oid umid, char *fdw_xact_id)
+{
+	int i;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		fxact = FdwXactCtl->fdw_xacts[i];
+		if (fxact->dbid == dbid &&
+			fxact->local_xid == xid &&
+			fxact->serverid == serverid &&
+			fxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->freeFdwXacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fxact = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fxact->fxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->numFdwXacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts++] = fxact;
+
+	fxact->held_by = InvalidBackendId;
+	fxact->dbid = dbid;
+	fxact->local_xid = xid;
+	fxact->serverid = serverid;
+	fxact->userid = userid;
+	fxact->umid = umid;
+	fxact->insert_start_lsn = InvalidXLogRecPtr;
+	fxact->insert_end_lsn = InvalidXLogRecPtr;
+	fxact->valid = false;
+	fxact->ondisk = false;
+	fxact->inredo = false;
+	memcpy(fxact->fdw_xact_id, fdw_xact_id, strlen(fdw_xact_id) + 1);
+
+	return fxact;
+}
+
+/*
+ * remove_fdw_xact
+ *
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdw_xact(FdwXact fdw_xact)
+{
+	int			cnt;
+
+	Assert(fdw_xact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Search the slot where this entry resided */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		if (FdwXactCtl->fdw_xacts[cnt] == fdw_xact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (cnt >= FdwXactCtl->numFdwXacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdw_xact->local_xid, fdw_xact->serverid, fdw_xact->userid)));
+
+	/* Remove the entry from active array */
+	FdwXactCtl->numFdwXacts--;
+	FdwXactCtl->fdw_xacts[cnt] = FdwXactCtl->fdw_xacts[FdwXactCtl->numFdwXacts];
+
+	/* Put it back into free list */
+	fdw_xact->fxact_free_next = FdwXactCtl->freeFdwXacts;
+	FdwXactCtl->freeFdwXacts = fdw_xact;
+
+	/* Reset informations */
+	fdw_xact->status = FDW_XACT_INITIAL;
+	fdw_xact->held_by = InvalidBackendId;
+	fdw_xact->fxact_next = NULL;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdw_xact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdw_xact->serverid;
+		record.dbid = fdw_xact->dbid;
+		record.xid = fdw_xact->local_xid;
+		record.userid = fdw_xact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdw_xact_remove));
+		recptr = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true and set FdwXactAtomicCommitReady to true if we require atomic commit.
+ * It is required if the transaction modified data on two or more servers including
+ * local node itself. This function also checks for each server if two-phase commit
+ * is enabled or not.
+ */
+static bool
+FdwXactAtomicCommitRequired(void)
+{
+	ListCell*	lc;
+	int			nserverswritten = 0;
+
+	if (!IsAtomicCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		/* Check if the foreign server is capable of two-phase commit protocol */
+		if (fdw_part->is_twophase_commit_enabled(fdw_part->server->serverid))
+			fdw_part->twophase_commit_enabled = true;
+		else if (fdw_part->modified)
+			MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	/* Atomic commit is required if we modified data on two or more participants */
+	if (nserverswritten <= 1)
+		return false;
+
+	FdwXactAtomicCommitReady = true;
+	return true;
+}
+
+bool
+FdwXactIsAtomicCommitReady(void)
+{
+	return FdwXactAtomicCommitReady;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * ForgetAllFdwXactParticipants
+ *
+ * Reset all the foreign transaction entries that this backend registered.
+ * If the foreign transaction has the corresponding FdwXact entry, resetting
+ * the held_by field means to leave that entry in unresolved state. If we
+ * leaves any entries, we update the oldest xmin of unresolved transaction
+ * so that transaction status of dangling transaction are not truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_lefts = 0;
+
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(cell, FdwXactAtomicCommitParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+
+		/* Skip if didn't register FdwXact entry yet */
+		if (fdw_part->fdw_xact == NULL)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in
+		 * FdwXactAtomicCommitParticipants could be used by other backend before we
+		 * forget in case where the resolver process removes the FdwXact entry
+		 * and other backend reuses it before we forget. So we need to check
+		 * if the entries are still associated with the transaction.
+		 */
+		if (fdw_part->fdw_xact->held_by == MyBackendId)
+		{
+			fdw_part->fdw_xact->held_by = InvalidBackendId;
+			n_lefts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Update the oldest local transaction of unresolved distributed
+	 * transaction if we leaved any FdwXact entries.
+	 */
+	if (n_lefts > 0)
+		FdwXactComputeRequiredXmin();
+
+	FdwXactAtomicCommitParticipants = NIL;
+}
+
+/*
+ * AtProcExit_FdwXact
+ *
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+/*
+ * Wait for foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDW_XACT_NOT_WAITING and then change
+ * that state to FDW_XACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDW_XACT_WAIT_COMPLETE once foreign transactions are resolved.
+ * This backend then resets its state to FDW_XACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue and changes the state to FDW_XACT_WAITING_RETRY.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+	ListCell	*lc;
+	List		*fdwxact_participants = NIL;
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsAtomicCommitRequested())
+		return;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDW_XACT_NOT_WAITING);
+
+	if (FdwXactAtomicCommitParticipants != NIL)
+	{
+		/*
+		 * If we're waiting for foreign transactions to be resolved that
+		 * we've prepared just before, use the participants list.
+		 */
+		Assert(MyPgXact->xid == wait_xid);
+		fdwxact_participants = FdwXactAtomicCommitParticipants;
+	}
+	else
+	{
+		/*
+		 * Get participants list from the global array. This is required (1)
+		 * when we're waiting for foreign transactions to be resolved that
+		 * is part of a local prepared transaction that is marked as prepared
+		 * during running, or (2) when we resolve the PREPARE'd distributed
+		 * transaction after restart.
+		 */
+		fdwxact_participants = get_fdw_xacts(MyDatabaseId, wait_xid,
+											 InvalidOid, InvalidOid, true);
+	}
+
+	/* Exit if we found no foreign transaction to resolve */
+	if (fdwxact_participants == NIL)
+		return;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	foreach(lc, fdwxact_participants)
+	{
+		FdwXact fdw_xact = (FdwXact) lfirst(lc);
+
+		/* Don't overwrite status if fate has been determined */
+		if (fdw_xact->status == FDW_XACT_PREPARED)
+			fdw_xact->status = (is_commit ?
+								FDW_XACT_COMMITTING_PREPARED :
+								FDW_XACT_ABORTING_PREPARED);
+	}
+
+	/* Set backend status and enqueue itself to the active queue*/
+	MyProc->fdwXactState = FDW_XACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	FdwXactQueueInsert();
+	LWLockRelease(FdwXactLock);
+
+	/* Launch a resolver process if not yet, or wake it up */
+	fdwxact_maybe_launch_resolver(false);
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDW_XACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDW_XACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDW_XACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+
+	/*
+	 * Forget the list of locked entries, also means that the entries
+	 * that could not resolved are remained as dangling transactions.
+	 */
+	ForgetAllFdwXactParticipants();
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Acquire FdwXactLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Insert MyProc into the tail of FdwXactActiveQueue.
+ */
+static void
+FdwXactQueueInsert(void)
+{
+	SHMQueueInsertBefore(&(FdwXactRslvCtl->FdwXactActiveQueue),
+						 &(MyProc->fdwXactLinks));
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * Resolve one distributed transaction. The target distributed transaction
+ * is fetched from either the active queue or the retry queue and its participants
+ * are fetched from either the global array.
+ *
+ * Release the waiter and return true if we resolved the all of the foreign
+ * transaction participants. On failure, we move the FdwXactLinks entry to the
+ * retry queue from the active queue, and raise an error and exit.
+ */
+bool
+FdwXactResolveDistributedTransaction(Oid dbid, bool is_active)
+{
+	FdwXactState	*state;
+	ListCell		*lc;
+	ListCell		*next;
+	PGPROC			*waiter = NULL;
+	List			*participants;
+	SHM_QUEUE		*target_queue;
+
+	if (is_active)
+		target_queue = &(FdwXactRslvCtl->FdwXactActiveQueue);
+	else
+		target_queue = &(FdwXactRslvCtl->FdwXactRetryQueue);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/* Fetch a waiter from beginning of the queue */
+	while ((waiter = (PGPROC *) SHMQueueNext(target_queue, target_queue,
+											 offsetof(PGPROC, fdwXactLinks))) != NULL)
+	{
+		/* Found a waiter */
+		if (waiter->databaseId == dbid)
+			break;
+	}
+
+	/* If no waiter, there is no job */
+	if (!waiter)
+	{
+		LWLockRelease(FdwXactLock);
+		return false;
+	}
+
+	Assert(TransactionIdIsValid(waiter->fdwXactWaitXid));
+
+	state = create_fdw_xact_state();
+	participants = get_fdw_xacts(dbid, waiter->fdwXactWaitXid, InvalidOid,
+								 InvalidOid, false);
+	LWLockRelease(FdwXactLock);
+
+	/* Resolve all foreign transactions one by one */
+	for (lc = list_head(participants); lc != NULL; lc = next)
+	{
+		FdwXact fdwxact = (FdwXact) lfirst(lc);
+
+		CHECK_FOR_INTERRUPTS();
+
+		next = lnext(lc);
+
+		state->serverid = fdwxact->serverid;
+		state->userid = fdwxact->userid;
+		state->umid = fdwxact->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		PG_TRY();
+		{
+			FdwXactResolveForeignTransaction(state, fdwxact, ERROR);
+		}
+		PG_CATCH();
+		{
+			/* Re-insert the waiter to the retry queue */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDW_XACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				SHMQueueInsertBefore(&(FdwXactRslvCtl->FdwXactRetryQueue),
+									 &(waiter->fdwXactLinks));
+				waiter->fdwXactState = FDW_XACT_WAITING_RETRY;
+			}
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved a foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+	}
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter
+	 * could already be detached if user cancelled to wait before
+	 * resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId	wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDW_XACT_WAIT_COMPLETE;
+
+		/* Wake up the waiter only when we have set state and removed from queue */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc xid %u", wait_xid);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	return true;
+}
+
+/*
+ * Resolve all dangling foreign transactions on the given database. Get
+ * all dangling foreign transactions from shmem global array and resolve
+ * them one by one.
+ */
+void
+FdwXactResolveAllDanglingTransactions(Oid dbid)
+{
+	List		*dangling_fdwxacts = NIL;
+	ListCell	*cell;
+	bool		n_resolved = 0;
+	int			i;
+
+	Assert(OidIsValid(dbid));
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/*
+	 * Walk over the global array to make the list of dangling transactions
+	 * of which corresponding local transaction is on the given database.
+	 */
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fxact = FdwXactCtl->fdw_xacts[i];
+
+		/*
+		 * Append the fdwxact entry on the given database to the list if
+		 * it's handled by nobody and the corresponding local transaction
+		 * is not part of the prepared transaction.
+		 */
+		if (fxact->dbid == dbid &&
+			fxact->held_by == InvalidBackendId &&
+			!TwoPhaseExists(fxact->local_xid))
+			dangling_fdwxacts = lappend(dangling_fdwxacts, fxact);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/* Return if there is no foreign transaction we need to resolve */
+	if (dangling_fdwxacts == NIL)
+		return;
+
+	foreach(cell, dangling_fdwxacts)
+	{
+		FdwXact fdwxact = (FdwXact) lfirst(cell);
+		FdwXactState *state;
+
+		state = create_fdw_xact_state();
+		state->serverid = fdwxact->serverid;
+		state->userid = fdwxact->userid;
+		state->umid = fdwxact->umid;
+		state->fdwxact_id = pstrdup(fdwxact->fdw_xact_id);
+
+		FdwXactResolveForeignTransaction(state, fdwxact, ERROR);
+
+		n_resolved++;
+	}
+
+	list_free(dangling_fdwxacts);
+
+	elog(DEBUG2, "resolved %d dangling foreign xacts", n_resolved);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ *
+ * In commit case, we have already prepared transactions on the foreign
+ * servers during pre-commit. And that prepared transactions will be
+ * resolved by the resolver process. So we don't do anything about the
+ * foreign transaction.
+ *
+ * In abort case, user requested rollback or we changed over rollback
+ * due to error during commit. To close current foreign transaction anyway
+ * we call rollback API to every foreign transaction. If we raised an error
+ * during preparing and came to here, it's possible that some entries of
+ * FdwXactParticipants already registered its FdwXact entry. If there is
+ * we leave them as dangling transaction and ask the resolver process to
+ * process them.
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		int left_fdwxacts = 0;
+		FdwXactState *state = create_fdw_xact_state();
+
+		foreach (lcell, FdwXactAtomicCommitParticipants)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * Count FdwXact entries that we registered to shared memory array
+			 * in this transaction.
+			 */
+			if (fdw_part->fdw_xact)
+			{
+				/*
+				 * The status of foreign transaction must be either preparing
+				 * or prepared. In any case, since we have registered FdwXact
+				 * entry we leave them to the resolver process. For the preparing
+				 * state, since the foreign transaction might not close yet we
+				 * fall through and call rollback API. For the prepared state,
+				 * since the foreign transaction has closed we don't need to do
+				 * anything.
+				 */
+				Assert(fdw_part->fdw_xact->status == FDW_XACT_PREPARING ||
+					   fdw_part->fdw_xact->status == FDW_XACT_PREPARED);
+
+				left_fdwxacts++;
+				if (fdw_part->fdw_xact->status == FDW_XACT_PREPARED)
+					continue;
+			}
+
+			state->serverid = fdw_part->server->serverid;
+			state->userid = fdw_part->usermapping->userid;
+			state->umid = fdw_part->usermapping->umid;
+			state->fdw_state = fdw_part->fdw_state;
+
+			/*
+			 * Rollback all current foreign transaction. Since we're rollbacking
+			 * the transaction it's too late even if we raise an error here.
+			 * So we log it as warning.
+			 */
+			if (!fdw_part->rollback_foreign_xact(state))
+				ereport(WARNING,
+						(errmsg("could not abort transaction on server \"%s\"",
+								fdw_part->server->servername)));
+		}
+
+		/* If we left some FdwXact entries, ask the resolver process */
+		if (left_fdwxacts > 0)
+		{
+			ereport(WARNING,
+					(errmsg("might have left %u foreign transactions in in-doubt status",
+							left_fdwxacts)));
+			fdwxact_maybe_launch_resolver(true);
+		}
+	}
+
+	ForgetAllFdwXactParticipants();
+	FdwXactAtomicCommitReady = false;
+}
+
+/*
+ * AtPrepare_FdwXacts
+ *
+ * If there are foreign servers involved in the transaction, this function
+ * prepares transactions on those servers.
+ *
+ * Note that it can happen that the transaction aborts after we prepared part
+ * of participants. In this case since we can change to abort we cannot forget
+ * FdwXactAtomicCommitParticipants here. These are processed by the resolver process
+ * during aborting, or at EOXact_FdwXacts.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (FdwXactAtomicCommitParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsAtomicCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when distributed_atomic_commit is \'disabled\'")));
+
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (FdwXactAtomicCommitRequired() &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_T_R_INTEGRITY_CONSTRAINT_VIOLATION),
+				 errmsg("can not prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * FdwXactResolveForeignTransaction
+ *
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * handler routine. The foreign transaction can be a dangling transaction
+ * that is not interested by nobody. If the fate of foreign transaction is
+ * not determined yet, it'sdetermined according to the status of corresponding
+ * local transaction.
+ *
+ * If the resolution is successful, remove the foreign transaction entry from
+ * the shared memory and also remove the corresponding on-disk file.
+ */
+static bool
+FdwXactResolveForeignTransaction(FdwXactState *state, FdwXact fdwxact,
+								 int elevel)
+{
+	ForeignServer		*server;
+	ForeignDataWrapper	*fdw;
+	FdwRoutine			*fdw_routine;
+	bool		is_commit;
+	bool		ret;
+
+	Assert(fdwxact);
+
+	/*
+	 * Determine whether we commit or abort this foreign transaction.
+	 */
+	if (fdwxact->status == FDW_XACT_COMMITTING_PREPARED)
+		is_commit = true;
+	else if (fdwxact->status == FDW_XACT_ABORTING_PREPARED)
+		is_commit = false;
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	else if (TransactionIdDidCommit(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_COMMITTING_PREPARED;
+		is_commit = true;
+	}
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+	{
+		fdwxact->status = FDW_XACT_ABORTING_PREPARED;
+		is_commit = false;
+	}
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+		is_commit = false;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction
+	 * state is neither committing or aborting. This should not
+	 * happen because we cannot determine to do commit or abort for
+	 * foreign transaction associated with the in-progress local
+	 * transaction.
+	 */
+	else
+		ereport(ERROR,
+				(errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid)));
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	/* Resolve the foreign transaction */
+	Assert(fdw_routine->ResolveForeignTransaction);
+
+	ret = fdw_routine->ResolveForeignTransaction(state, is_commit);
+
+	if (!ret)
+	{
+		ereport(elevel,
+				(errmsg("could not %s a prepared foreign transaction on server \"%s\"",
+						is_commit ? "commit" : "rollback", server->servername),
+				 errdetail("local transaction id is %u, connected by user id %u",
+						   fdwxact->local_xid, fdwxact->userid)));
+	}
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdw_xact(fdwxact);
+	LWLockRelease(FdwXactLock);
+
+	return ret;
+}
+
+static FdwXactState *
+create_fdw_xact_state(void)
+{
+	FdwXactState *state;
+
+	state = palloc(sizeof(FdwXactState));
+	state->serverid = InvalidOid;
+	state->userid = InvalidOid;
+	state->umid = InvalidOid;
+	state->fdwxact_id = NULL;
+	state->fdw_state = NULL;
+
+	return state;
+}
+
+/*
+ * Return one FdwXact entry that matches to given arguments, otherwise
+ * return NULL. Since this function search FdwXact entry by unique key
+ * all arguments should be valid.
+ */
+static FdwXact
+get_one_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				 bool need_lock)
+{
+	List	*fdw_xact_list;
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, need_lock);
+
+	/* Could not find entry */
+	if (fdw_xact_list == NIL)
+		return NULL;
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdw_xact_list) == 1);
+
+	return (FdwXact) linitial(fdw_xact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdw_xact_exists(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdw_xact_list;
+
+	fdw_xact_list = get_fdw_xacts(dbid, xid, serverid, userid, true);
+
+	return fdw_xact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_prepared_fdw_xacts.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdw_xacts(int *length)
+{
+	List		*all_fdw_xacts;
+	ListCell	*lc;
+	FdwXact		fdw_xacts;
+	int			num_fdw_xacts = 0;
+
+	Assert(length != NULL);
+
+	/* Get all entries */
+	all_fdw_xacts = get_fdw_xacts(InvalidOid, InvalidTransactionId,
+								  InvalidOid, InvalidOid, true);
+
+	if (all_fdw_xacts == NIL)
+	{
+		*length = 0;
+		return NULL;
+	}
+
+	fdw_xacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdw_xacts));
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdw_xacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdw_xacts + num_fdw_xacts, fx,
+			   sizeof(FdwXactData));
+		num_fdw_xacts++;
+	}
+
+	*length = num_fdw_xacts;
+	list_free(all_fdw_xacts);
+
+	return fdw_xacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return
+ * NIL.
+ */
+static List*
+get_fdw_xacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			  bool need_lock)
+{
+	int i;
+	List	*fdw_xact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact	fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool	matches = true;
+
+		/* xid */
+		if (xid != InvalidTransactionId && xid != fdw_xact->local_xid)
+			matches = false;
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdw_xact->dbid != dbid)
+			matches = false;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdw_xact->serverid)
+			matches = false;
+
+		/* userid */
+		if (OidIsValid(userid) && fdw_xact->userid != userid)
+			matches = false;
+
+		/* Append it if matched */
+		if (matches)
+			fdw_xact_list = lappend(fdw_xact_list, fdw_xact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdw_xact_list;
+}
+
+/*
+ * fdw_xact_redo
+ * Apply the redo log for a foreign transaction.
+ */
+void
+fdw_xact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDW_XACT_REMOVE)
+	{
+		xl_fdw_xact_remove *record = (xl_fdw_xact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier with in the form
+ * of "fx_<random number>_<xid>_<serverid>_<userid> whose length is always
+ * less than NAMEDATALEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+generate_fdw_xact_identifier(TransactionId xid, Oid serverid, Oid userid)
+{
+	char*	fdw_xact_id;
+
+	fdw_xact_id = (char *)palloc0(FDW_XACT_ID_MAX_LEN * sizeof(char));
+
+	snprintf(fdw_xact_id, FDW_XACT_ID_MAX_LEN, "%s_%ld_%u_%d_%d",
+			 "fx", Abs(random()), xid, serverid, userid);
+	fdw_xact_id[strlen(fdw_xact_id)] = '\0';
+
+	return fdw_xact_id;
+}
+
+/*
+ * CheckPointFdwXact
+ *
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * In order to avoid disk I/O while holding a light weight lock, the function
+ * first collects the files which need to be synced under FdwXactLock and then
+ * syncs them after releasing the lock. This approach creates a race condition:
+ * after releasing the lock, and before syncing a file, the corresponding
+ * foreign transaction entry and hence the file might get removed. The function
+ * checks whether that's true and ignores the error if so.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdw_xacts = 0;
+
+	/* Quick get-away, before taking lock */
+	if (max_prepared_foreign_xacts <= 0)
+		return;
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	/* Another quick, before we allocate memory */
+	if (FdwXactCtl->numFdwXacts <= 0)
+	{
+		LWLockRelease(FdwXactLock);
+		return;
+	}
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	for (cnt = 0; cnt < FdwXactCtl->numFdwXacts; cnt++)
+	{
+		FdwXact		fxact = FdwXactCtl->fdw_xacts[cnt];
+
+		if ((fxact->valid || fxact->inredo) &&
+			!fxact->ondisk &&
+			fxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fxact->dbid, fxact->local_xid,
+								fxact->serverid, fxact->userid,
+								buf, len);
+			fxact->ondisk = true;
+			fxact->insert_start_lsn = InvalidXLogRecPtr;
+			fxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdw_xacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDW_XACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdw_xacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdw_xacts,
+							 serialized_fdw_xacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDW_XACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDW_XACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * ProcessFdwXactBuffer
+ *
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid = ShmemVariableCache->nextXid;
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(insert_start_lsn != InvalidXLogRecPtr);
+
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid, true);
+		if (buf == NULL)
+		{
+			ereport(WARNING,
+					(errmsg("removing corrupt fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+			return NULL;
+		}
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return thecontents in
+ * a structure allocated in-memory. Otherwise return NULL. The structure can
+ * be later freed by the caller.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				bool give_warnings)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+	{
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not stat FDW transaction state file \"%s\": %m",
+							path)));
+		return NULL;
+	}
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdw_xact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+	{
+		CloseTransientFile(fd);
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+		return NULL;
+	}
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+	{
+		CloseTransientFile(fd);
+		return NULL;
+	}
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDW_XACT_FILE_READ);
+	if (read(fd, buf, stat.st_size) != stat.st_size)
+	{
+		pgstat_report_wait_end();
+		CloseTransientFile(fd);
+		if (give_warnings)
+			ereport(WARNING,
+					(errcode_for_file_access(),
+					 errmsg("could not read FDW transaction state file \"%s\": %m",
+					  path)));
+		return NULL;
+	}
+
+	pgstat_report_wait_end();
+	CloseTransientFile(fd);
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+	{
+		pfree(buf);
+		return NULL;
+	}
+
+	/* Check if the contents is an expected data */
+	fxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fxact_file_data->dbid  != dbid ||
+		fxact_file_data->serverid != serverid ||
+		fxact_file_data->userid != userid ||
+		fxact_file_data->local_xid != xid)
+	{
+		ereport(WARNING,
+			(errmsg("invalid foreign transaction state file \"%s\"",
+					path)));
+		CloseTransientFile(fd);
+		pfree(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+/*
+ * PrescanFdwXacts
+ *
+ * Scan the all foreign transactions directory for oldest active transaction.
+ * This is run during database startup, after we completed reading WAL.
+ * ShmemVariableCache->nextXid has been set to one more than the highest XID
+ * for which evidence exists in WAL.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	TransactionId nextXid = ShmemVariableCache->nextXid;
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+		 strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			TransactionId local_xid;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/*
+			 * Remove a foreign prepared transaction file corresponding to an
+			 * XID, which is too new.
+			 */
+			if (TransactionIdFollowsOrEquals(local_xid, nextXid))
+			{
+				ereport(WARNING,
+						(errmsg("removing future foreign prepared transaction file \"%s\"",
+								clde->d_name)));
+				RemoveFdwXactFile(dbid, local_xid, serverid, userid, true);
+				continue;
+			}
+
+			if (TransactionIdPrecedesOrEquals(local_xid, oldestActiveXid))
+				oldestActiveXid = local_xid;
+		}
+	}
+
+	FreeDir(cldir);
+	return oldestActiveXid;
+}
+
+/*
+ * restoreFdwXactData
+ *
+ * Scan pg_fdw_xact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDW_XACTS_DIR);
+	while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid, bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * FdwXactRedoAdd
+ *
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fxact = insert_fdw_xact(fxact_data->dbid, fxact_data->local_xid,
+							fxact_data->serverid, fxact_data->userid,
+							fxact_data->umid, fxact_data->fdw_xact_id);
+
+	/*
+	 * Set status as preparing, since we do not know the xact status
+	 * right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fxact->status = FDW_XACT_PREPARING;
+	fxact->insert_start_lsn = start_lsn;
+	fxact->insert_end_lsn = end_lsn;
+	fxact->inredo = true;	/* added in redo */
+	fxact->valid = false;
+	fxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * FdwXactRedoRemove
+ *
+ * Remove the corresponding fdw_xact entry from FdwXactCtl.
+ * Also remove fdw_xact file if a foreign transaction was saved
+ * via an earlier checkpoint.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdw_xact(dbid, xid, serverid, userid,
+							   false);
+
+	if (fdwxact == NULL)
+		return;
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdw_xact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdw_xacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_distributed_atomic_commit(int *newval, void **extra, GucSource source)
+{
+	DistributedAtomicCommitLevel newDistributedAtomicCommitLevel = *newval;
+
+		/* Parameter check */
+	if (newDistributedAtomicCommitLevel > DISTRIBUTED_ATOMIC_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"distributed_atomic_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdw_xacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_prepared_fdw_xacts(PG_FUNCTION_ARGS)
+{
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdw_xacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdw_xacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(6, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdw_xacts = get_all_fdw_xacts(&num_fdw_xacts);
+		status->num_xacts = num_fdw_xacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdw_xact = &status->fdw_xacts[status->cur_xact++];
+		Datum		values[6];
+		bool		nulls[6];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdw_xact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdw_xact->dbid);
+		values[1] = TransactionIdGetDatum(fdw_xact->local_xid);
+		values[2] = ObjectIdGetDatum(fdw_xact->serverid);
+		values[3] = ObjectIdGetDatum(fdw_xact->userid);
+		switch (fdw_xact->status)
+		{
+			case FDW_XACT_PREPARING:
+				xact_status = "prepared";
+				break;
+			case FDW_XACT_COMMITTING_PREPARED:
+				xact_status = "committing";
+				break;
+			case FDW_XACT_ABORTING_PREPARED:
+				xact_status = "aborting";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		/* should this be really interpreted by FDW */
+		values[5] = PointerGetDatum(cstring_to_text_with_len(fdw_xact->fdw_xact_id,
+															 strlen(fdw_xact->fdw_xact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXactState *state;
+	UserMapping		*usermapping;
+	FdwXact			fdwxact;
+	bool			ret;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, true);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	usermapping = GetUserMapping(userid, serverid);
+
+	state = create_fdw_xact_state();
+	state->serverid = serverid;
+	state->userid = userid;
+	state->umid = usermapping->umid;
+
+	ret = FdwXactResolveForeignTransaction(state, fdwxact, LOG);
+
+	PG_RETURN_BOOL(ret);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_fdw_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdw_xact(MyDatabaseId, xid, serverid, userid, false);
+	if (fdwxact == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("could not find foreign transaction entry"))));
+
+	remove_fdw_xact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_VOID();
+}
diff --git a/src/backend/access/fdwxact/fdwxact_launcher.c b/src/backend/access/fdwxact/fdwxact_launcher.c
new file mode 100644
index 0000000..39f351b
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact_launcher.c
@@ -0,0 +1,641 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * There is a shared memory area where the information of resolver process
+ * is stored. Requesting of starting new resolver process by backend process
+ * is done via that shared memory area. Note that the launcher is assuming
+ * that there is no more than one starting request for a database.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/fdwxact_launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launcher_sigusr2(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid, int slot);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+Datum pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS);
+
+/*
+ * Wake up the launcher process to retry launch. This is used by
+ * the resolver process is being stopped.
+ */
+void
+FdwXactLauncherWakeupToRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request resolution. This is
+ * used by the backend process.
+ */
+void
+FdwXactLauncherWakeupToRequest(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->FdwXactActiveQueue));
+		SHMQueueInit(&(FdwXactRslvCtl->FdwXactRetryQueue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR1: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a foreign_xact_resolution_retry_interval
+		 * but always try to start by the backend request.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool launched;
+
+			ResetLatch(MyLatch);
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher launch",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver worker
+ * if not running yet. A foreign transaction resolver worker is responsible
+ * for resolution of foreign transaction that are registered on a database.
+ * So if a resolver worker already is launched, we don't need to launch new
+ * one.
+ */
+void
+fdwxact_maybe_launch_resolver(bool ignore_error)
+{
+	FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->pid != InvalidPid &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/*
+	 * If we found the resolver for my database, we don't need to launch new
+	 * one but wake running worker up.
+	 */
+	if (found)
+	{
+		SetLatch(resolver->latch);
+
+		elog(DEBUG1, "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		return;
+	}
+
+	/* Looking for unused resolver slot */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	/*
+	 * However if there are no more free worker slots, inform user about it before
+	 * exiting.
+	 */
+	if (!found)
+	{
+		LWLockRelease(FdwXactResolverLock);
+
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+		return;
+	}
+
+	Assert(resolver->pid == InvalidPid);
+
+	/* Found a new resolver process */
+	resolver->dbid = MyDatabaseId;
+	resolver->in_use = true;
+
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Wake up launcher */
+	FdwXactLauncherWakeupToRequest();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid' at 'slot' if given. If slot is negative value we find an unused slot.
+ * Note that caller must hold FdwXactResolverLock in exclusive mode.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid, int slot)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int launch_slot = slot;
+
+	/* If slot number is invalid, we find an unused slot */
+	if (launch_slot < 0)
+	{
+		int i;
+
+		for (i = 0; i < max_foreign_xact_resolvers; i++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+			if (resolver->in_use && resolver->dbid == dbid)
+				return;
+
+			if (!resolver->in_use)
+			{
+				launch_slot = i;
+				break;
+			}
+		}
+	}
+
+	/* No unused found */
+	if (launch_slot < 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[launch_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_main_arg = Int32GetDatum(launch_slot);
+	bgw.bgw_notify_pid = (Datum) 0;
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch all foreign transaction resolvers that are required by backend process
+ * but not running. Return true if we launch any resolver.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	int i, j;
+	int num_launches = 0;
+	int num_unused_slots = 0;
+	int num_dbs = 0;
+	bool launched = false;
+	Oid	*dbs_to_launch;
+	Oid	*dbs_having_worker = palloc0(sizeof(Oid) * max_foreign_xact_resolvers);
+
+	/*
+	 * Launch resolver workers on the databases that are requested
+	 * by backend processes while looking unused slots.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* Remember unused worker slots */
+		if (!resolver->in_use)
+		{
+			num_unused_slots++;
+			continue;
+		}
+
+		/* Remember databases that are having a resolve worker, fall through */
+		if (OidIsValid(resolver->dbid))
+			dbs_having_worker[num_dbs++] = resolver->dbid;
+
+		/* Launch the backend-requested worker */
+		if (resolver->in_use &&
+			OidIsValid(resolver->dbid) &&
+			resolver->pid == InvalidPid)
+		{
+			fdwxact_launch_resolver(resolver->dbid, i);
+			launched = true;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* quick exit if no unused slot */
+	if (num_unused_slots == 0)
+		return launched;
+
+	/*
+	 * Launch the stopped resolver on the database that has unresolved
+	 * foreign transaction but doesn't have any resolver. Scanning
+	 * all FdwXact entries could take time but it's harmless for the
+	 * relaunch case.
+	 */
+	dbs_to_launch = (Oid *) palloc(sizeof(Oid) * num_unused_slots);
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->numFdwXacts; i++)
+	{
+		FdwXact fdw_xact = FdwXactCtl->fdw_xacts[i];
+		bool found = false;
+
+		/* unused slot is full */
+		if (num_launches > num_unused_slots)
+			break;
+
+		for (j = 0; j < num_dbs; j++)
+		{
+			if (dbs_having_worker[j] == fdw_xact->dbid)
+			{
+				found = true;
+				break;
+			}
+		}
+
+		/* Register the database if any resolvers aren't working on that */
+		if (!found)
+			dbs_to_launch[num_launches++] = fdw_xact->dbid;
+	}
+
+	/* Launch resolver process for a database at any worker slot */
+	for (i = 0; i < num_launches; i++)
+	{
+		fdwxact_launch_resolver(dbs_to_launch[i], -1);
+		launched = true;
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+
+/*
+ * Returns activity of foreign transaction resolvers, including pids, the number
+ * of tasks and the last resolution time.
+ */
+Datum
+pg_stat_get_fdwxact_resolver(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/fdwxact_resolver.c b/src/backend/access/fdwxact/fdwxact_resolver.c
new file mode 100644
index 0000000..0b754da
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact_resolver.c
@@ -0,0 +1,331 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for every databases.
+ *
+ * A resolver process continues to resolve foreign transactions on a database
+ * It resolves two types of foreign transactions: on-line foreign transaction
+ * and dangling foreign transaction. The on-line foreign transaction is a
+ * foreign transaction that a concurrent backend process is waiting for
+ * resolution. The dangling transaction is a foreign transaction that corresponding
+ * distributed transaction ended up in in-doubt state. A resolver process
+ * doesn' exit as long as there is at least one unresolved foreign transaction
+ * on the database even if the timeout has come.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/fdwxact_resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+
+//static MemoryContext ResolveContext = NULL;
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FdwXactRslvLoop(void);
+static long FdwXactRslvComputeSleepTime(TimestampTz now);
+static void FdwXactRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+	FdwXactLauncherWakeupToRetry();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	TIMESTAMP_NOBEGIN(MyFdwXactResolver->last_resolved_time);
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FdwXactRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FdwXactRslvLoop(void)
+{
+	TimestampTz last_retry_time = 0;
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time;
+		bool		resolved;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		/* Resolve one distributed transaction */
+		StartTransactionCommand();
+		resolved = FdwXactResolveDistributedTransaction(MyDatabaseId, true);
+		CommitTransactionCommand();
+
+		now = GetCurrentTimestamp();
+
+		/* Update my state */
+		if (resolved)
+			MyFdwXactResolver->last_resolved_time = now;
+
+		if (TimestampDifferenceExceeds(last_retry_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			StartTransactionCommand();
+			resolved = FdwXactResolveDistributedTransaction(MyDatabaseId, false);
+			CommitTransactionCommand();
+
+			last_retry_time = GetCurrentTimestamp();
+
+			/* Update my state */
+			if (resolved)
+				MyFdwXactResolver->last_resolved_time = last_retry_time;
+		}
+
+		/* Check for fdwxact resolver timeout */
+		FdwXactRslvCheckTimeout(now);
+
+		/*
+		 * If we have resolved any distributed transaction we go the next
+		 * without both resolving dangling transaction and sleeping because
+		 * there might be other on-line transactions waiting to be resolved.
+		 */
+		if (!resolved)
+		{
+			/* Resolve dangling transactions as mush as possible */
+			StartTransactionCommand();
+			FdwXactResolveAllDanglingTransactions(MyDatabaseId);
+			CommitTransactionCommand();
+
+			sleep_time = FdwXactRslvComputeSleepTime(now);
+
+			MemoryContextResetAndDeleteChildren(resolver_ctx);
+			MemoryContextSwitchTo(TopMemoryContext);
+
+			rc = WaitLatch(MyLatch,
+						   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+						   sleep_time,
+						   WAIT_EVENT_FDW_XACT_RESOLVER_MAIN);
+
+			if (rc & WL_POSTMASTER_DEATH)
+				proc_exit(1);
+		}
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FdwXactRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	/*
+	 * Reached to the timeout. We exit if there is no more both pending on-line
+	 * transactions and dangling transactions.
+	 */
+	if (!fdw_xact_exists(InvalidTransactionId, MyDatabaseId, InvalidOid,
+						 InvalidOid))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyFdwXactResolver->dbid))));
+		CommitTransactionCommand();
+
+		fdwxact_resolver_detach();
+		proc_exit(0);
+	}
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. Return the sleep time
+ * in milliseconds, -1 means that we reached to the timeout and should exits
+ */
+static long
+FdwXactRslvComputeSleepTime(TimestampTz now)
+{
+	static TimestampTz	wakeuptime = 0;
+	long	sleeptime;
+	long	sec_to_timeout;
+	int		microsec_to_timeout;
+
+	if (now >= wakeuptime)
+		wakeuptime = TimestampTzPlusMilliseconds(now,
+												 foreign_xact_resolution_retry_interval);
+
+	/* Compute relative time until wakeup. */
+	TimestampDifference(now, wakeuptime,
+						&sec_to_timeout, &microsec_to_timeout);
+
+	sleeptime = sec_to_timeout * 1000 + microsec_to_timeout / 1000;
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index c2db19b..fb63471 100644
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -2629,10 +2629,6 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 		heap_freetuple(heaptup);
 	}
 
-	/* Make note that we've wrote on non-temprary relation */
-	if (RelationNeedsWAL(relation))
-		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
-
 	return HeapTupleGetOid(tup);
 }
 
@@ -3457,10 +3453,6 @@ l1:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
-	/* Make note that we've wrote on non-temprary relation */
-	if (RelationNeedsWAL(relation))
-		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
-
 	return HeapTupleMayBeUpdated;
 }
 
@@ -4411,10 +4403,6 @@ l2:
 	if (old_key_tuple != NULL && old_key_copied)
 		heap_freetuple(old_key_tuple);
 
-	/* Make note that we've wrote on non-temprary relation */
-	if (RelationNeedsWAL(relation))
-		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
-
 	bms_free(hot_attrs);
 	bms_free(proj_idx_attrs);
 	bms_free(key_attrs);
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index 5514db1..742e825 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -8,9 +8,9 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
-	   gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
-	   mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
-	   smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdwxactdesc.o \
+	genericdesc.o  gindesc.o gistdesc.o hashdesc.o heapdesc.o \
+	logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \
+	seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000..7061bba
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,65 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdw_xactdesc.c
+ *		PostgreSQL distributed transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdw_xactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdw_xact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDW_XACT_INSERT)
+	{
+		FdwXactOnDiskData *fdw_insert_xlog = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_insert_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_insert_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_insert_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_insert_xlog->local_xid);
+		/* TODO: This should be really interpreted by each FDW */
+
+		/*
+		 * TODO: we also need to assess whether we want to add this
+		 * information
+		 */
+		appendStringInfo(buf, " foreign transaction info: %s",
+						 fdw_insert_xlog->fdw_xact_id);
+	}
+	else
+	{
+		xl_fdw_xact_remove *fdw_remove_xlog = (xl_fdw_xact_remove *) rec;
+
+		appendStringInfo(buf, "Foreign server oid: %u", fdw_remove_xlog->serverid);
+		appendStringInfo(buf, " user oid: %u", fdw_remove_xlog->userid);
+		appendStringInfo(buf, " database id: %u", fdw_remove_xlog->dbid);
+		appendStringInfo(buf, " local xid: %u", fdw_remove_xlog->xid);
+	}
+
+}
+
+const char *
+fdw_xact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDW_XACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDW_XACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 00741c7..4a9ab3d 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -112,14 +112,16 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_prepared_xacts=%d max_locks_per_xact=%d "
 						 "wal_level=%s wal_log_hints=%s "
-						 "track_commit_timestamp=%s",
+						 "track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_prepared_xacts,
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47..f15c83a 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -12,9 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
-	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
-	xact.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o generic_xlog.o multixact.o \
+	parallel.o rmgr.o slru.o subtrans.o timeline.o transam.o twophase.o \
+	twophase_rmgr.o varsup.o xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9368b56..8b360b1 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -9,6 +9,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
 #include "access/generic_xlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 3942734..bc4e109 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -844,6 +845,35 @@ TwoPhaseGetGXact(TransactionId xid)
 }
 
 /*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
+/*
  * TwoPhaseGetDummyProc
  *		Get the dummy backend ID for prepared transaction specified by XID
  *
@@ -2316,6 +2346,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2375,6 +2411,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index d967400..1d06e0a 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1131,6 +1132,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1139,6 +1141,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsAtomicCommitReady();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1177,12 +1180,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1340,6 +1344,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -1994,6 +2006,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2150,6 +2165,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2237,6 +2253,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2426,6 +2444,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2631,6 +2650,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7eed586..cce4fd4 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
 #include "access/subtrans.h"
@@ -5250,6 +5251,7 @@ BootStrapXLOG(void)
 	ControlFile->MaxConnections = MaxConnections;
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6337,6 +6339,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -6861,14 +6866,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdw_xact, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -7060,7 +7066,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7566,6 +7575,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7884,6 +7894,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9200,6 +9213,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9633,7 +9647,8 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9665,6 +9680,7 @@ XLogReportParameters(void)
 		ControlFile->MaxConnections = MaxConnections;
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9870,6 +9886,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -10068,6 +10085,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->MaxConnections = xlrec.MaxConnections;
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 715995d..1b9cdbb 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -291,6 +291,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_prepared_fdw_xacts AS
+       SELECT * FROM pg_prepared_fdw_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
 	l.objoid, l.classoid, l.objsubid,
@@ -773,6 +776,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_fdwxact_resolvers AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_fdwxact_resolver() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index b58a74f..4d4c339 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2504,9 +2504,16 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
 
+	}
+
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
 
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index e5dd995..6056feb 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
@@ -1093,6 +1094,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1407,6 +1420,16 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
 	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srv->serverid,
+						useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
+	/*
 	 * Do the deletion
 	 */
 	object.classId = UserMappingRelationId;
@@ -1559,6 +1582,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index 1e72e9f..2fee05d 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "catalog/partition.h"
 #include "catalog/pg_inherits.h"
 #include "catalog/pg_type.h"
@@ -749,7 +750,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	MemoryContextSwitchTo(oldContext);
 
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 5d2cd0e..71c9916 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,10 +226,33 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
 
+	}
+
 	return scanstate;
 }
 
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index e2836b7..2557568 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "commands/trigger.h"
@@ -44,6 +45,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "storage/bufmgr.h"
@@ -485,6 +487,10 @@ ExecInsert(ModifyTableState *mtstate,
 								HEAP_INSERT_SPECULATIVE,
 								NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self),
 												   estate, true, &specConflict,
@@ -530,6 +536,10 @@ ExecInsert(ModifyTableState *mtstate,
 								estate->es_output_cid,
 								0, NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, &(tuple->t_self),
@@ -722,6 +732,11 @@ ldelete:;
 							 true /* wait for commit */ ,
 							 &hufd,
 							 changingPart);
+
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case HeapTupleSelfUpdated:
@@ -1210,6 +1225,11 @@ lreplace:;
 							 estate->es_crosscheck_snapshot,
 							 true /* wait for commit */ ,
 							 &hufd, &lockmode);
+
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case HeapTupleSelfUpdated:
@@ -2315,6 +2335,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index a0bcc04..b2097ad 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -155,6 +155,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMapping - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index d2b695e..b722b9a 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -15,6 +15,8 @@
 #include <unistd.h>
 
 #include "libpq/pqsignal.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 42bccce..5116369 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3484,6 +3484,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3678,6 +3684,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDW_XACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -3893,6 +3902,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDW_XACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDW_XACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index cb49f32..d9faef0 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -896,6 +898,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -971,12 +977,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index afb4972..960fd6a 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -154,6 +154,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDW_XACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 0c86a58..c5610ee 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -150,6 +152,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
 		size = add_size(size, BackendRandomShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -270,6 +274,8 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 	SyncScanShmemInit();
 	AsyncShmemInit();
 	BackendRandomShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index dc7e875..48bb87a 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -91,6 +91,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -246,6 +248,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1327,6 +1330,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1392,6 +1396,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1442,6 +1447,15 @@ GetOldestXmin(Relation rel, int flags)
 		result = replication_slot_xmin;
 
 	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDW_XACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
+	/*
 	 * After locks have been released and defer_cleanup_age has been applied,
 	 * check whether we need to back up further to make logical decoding
 	 * possible. We need to do so if we're computing the global limit (rel =
@@ -3030,6 +3044,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index e6025ec..a42d06e 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -50,3 +50,5 @@ OldSnapshotTimeMapLock				42
 BackendRandomLock					43
 LogicalRepWorkerLock				44
 CLogTruncationLock					45
+FdwXactLock					46
+FdwXactResolverLock			47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 6ad5044..6e7b3b8 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -405,6 +406,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* initialize fields for fdw xact */
+	MyProc->fdwXactState = FDW_XACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -806,6 +811,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index a3b9757..48f3c59 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -2994,6 +2996,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 0327b29..ffdc166 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/transam.h"
@@ -377,6 +378,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 };
 
 /*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry distributed_atomic_commit_options[] = {
+	{"required", DISTRIBUTED_ATOMIC_COMMIT_REQUIRED, false},
+	{"prefer", DISTRIBUTED_ATOMIC_COMMIT_PREFER, false},
+	{"disabled", DISTRIBUTED_ATOMIC_COMMIT_DISABLED, false},
+	{"on", DISTRIBUTED_ATOMIC_COMMIT_REQUIRED, false},
+	{"off", DISTRIBUTED_ATOMIC_COMMIT_DISABLED, false},
+	{"true", DISTRIBUTED_ATOMIC_COMMIT_REQUIRED, true},
+	{"false", DISTRIBUTED_ATOMIC_COMMIT_DISABLED, true},
+	{"yes", DISTRIBUTED_ATOMIC_COMMIT_REQUIRED, true},
+	{"no", DISTRIBUTED_ATOMIC_COMMIT_DISABLED, true},
+	{"1", DISTRIBUTED_ATOMIC_COMMIT_REQUIRED, true},
+	{"0", DISTRIBUTED_ATOMIC_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
+/*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
  */
@@ -658,6 +678,10 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2234,6 +2258,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, RESOURCES_ASYNCHRONOUS,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4054,6 +4124,16 @@ static struct config_enum ConfigureNamesEnum[] =
 	},
 
 	{
+		{"distributed_atomic_commit", PGC_USERSET, FDWXACT_SETTINGS,
+		 gettext_noop("Use of distributed atomic commit for the current transaction."),
+			NULL
+		},
+		&distributed_atomic_commit,
+		DISTRIBUTED_ATOMIC_COMMIT_DISABLED, distributed_atomic_commit_options,
+		check_distributed_atomic_commit, NULL, NULL
+	},
+
+	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
 			NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 3fe257c..88387ed 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -121,6 +121,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -287,6 +289,20 @@
 
 
 #------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
+#------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index ad06e8e..ca3eb62 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index ab5cb7f..609578c 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -209,6 +209,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdw_xact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 895a51f..7df88e0 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -306,6 +306,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_worker_processes);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 6fb403a..6d867c8 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -730,6 +730,7 @@ GuessControlValues(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -957,6 +958,7 @@ RewriteControlFile(void)
 	ControlFile.MaxConnections = 100;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* Contents are protected with a CRC */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca..b616cea 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000..b4de88b
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,149 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL distributed transaction manager
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDW_XACT_H
+#define FDW_XACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+#define	FDW_XACT_NOT_WAITING		0
+#define	FDW_XACT_WAITING			1
+#define	FDW_XACT_WAITING_RETRY		2
+#define	FDW_XACT_WAIT_COMPLETE		3
+
+#define FdwXactEnabled() (max_prepared_foreign_xacts > 0)
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDW_XACT_ID_MAX_LEN 200
+
+/* Enum to track the status of prepared foreign transaction */
+typedef enum
+{
+	FDW_XACT_INITIAL,
+	FDW_XACT_PREPARING,					/* foreign transaction is being prepared */
+	FDW_XACT_PREPARED,					/* foreign transaction is prepared */
+	FDW_XACT_COMMITTING_PREPARED,		/* foreign prepared transaction is to
+										 * be committed */
+	FDW_XACT_ABORTING_PREPARED, /* foreign prepared transaction is to be
+								 * aborted */
+} FdwXactStatus;
+
+
+/* Enum for distributed_atomic_commit parameter */
+typedef enum
+{
+	DISTRIBUTED_ATOMIC_COMMIT_DISABLED,	/* disable distributed atomic commit */
+	DISTRIBUTED_ATOMIC_COMMIT_PREFER,	/* use twophase commit where available */
+	DISTRIBUTED_ATOMIC_COMMIT_REQUIRED	/* all foreign servers have to support twophase
+										 * commit */
+} DistributedAtomicCommitLevel;
+
+/* Shared memory entry for a prepared or being prepared foreign transaction */
+typedef struct FdwXactData *FdwXact;
+
+typedef struct FdwXactData
+{
+	FdwXact		fxact_free_next;	/* Next free FdwXact entry */
+	FdwXact		fxact_next;			/* Pointer to the neext FdwXact entry accosiated
+									 * with the same transaction */
+	Oid				dbid;			/* database oid where to find foreign server
+									 * and user mapping */
+	TransactionId	local_xid;		/* XID of local transaction */
+	Oid				serverid;		/* foreign server where transaction takes place */
+	Oid				userid;			/* user who initiated the foreign transaction */
+	Oid				umid;
+	FdwXactStatus 	status;			/* The state of the foreign transaction. This
+									 * doubles as the action to be taken on this entry. */
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;		/* XLOG offset of inserting this entry start */
+	XLogRecPtr	insert_end_lsn;		/* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN];		/* prepared transaction identifier */
+} FdwXactData;
+
+/* Shared memory layout for maintaining foreign prepared transaction entries. */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		freeFdwXacts;
+
+	/* Number of valid foreign transaction entries */
+	int			numFdwXacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdw_xacts[FLEXIBLE_ARRAY_MEMBER];		/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+typedef struct FdwXactState
+{
+	Oid		serverid;
+	Oid		userid;
+	Oid		umid;
+	char	*fdwxact_id;
+	void	*fdw_state;		/* foreign-data wrapper can keep state here */
+} FdwXactState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	distributed_atomic_commit;
+
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern bool fdw_xact_exists(TransactionId xid, Oid dboid, Oid serverid,
+				Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern void PreCommit_FdwXacts(void);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern bool FdwXactResolveDistributedTransaction(Oid dbid, bool is_active);
+extern void FdwXactResolveAllDanglingTransactions(Oid dbid);
+extern bool FdwXactIsAtomicCommitReady(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void RegisterFdwXactByRelId(Oid relid, bool modified);
+extern void RegisterFdwXactByServerId(Oid serverid, bool modified);
+extern void FdwXactMarkForeignServerAccessed(Oid relid, bool modified);
+extern bool check_distributed_atomic_commit(int *newval, void **extra,
+										  GucSource source);
+
+#endif   /* FDW_XACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000..4ea65b2
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,32 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _FDWXACT_LAUNCHER_H
+#define _FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherWakeupToRequest(void);
+extern void FdwXactLauncherWakeupToRetry(void);
+
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+
+extern bool IsFdwXactLauncher(void);
+
+extern void fdwxact_maybe_launch_resolver(bool ignore_error);
+
+
+#endif	/* _FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000..6b2a24f
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int foreign_xact_resolver_timeout;
+
+#endif		/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000..e92b5a1
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,52 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDW_XACT_INSERT	0x00
+#define XLOG_FDW_XACT_REMOVE	0x10
+
+/* Same as GIDSIZE */
+#define FDW_XACT_MAX_ID_LEN 200
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdw_xact_id[FDW_XACT_MAX_ID_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdw_xact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+} xl_fdw_xact_remove;
+
+extern void fdw_xact_redo(XLogReaderState *record);
+extern void fdw_xact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdw_xact_identify(uint8 info);
+
+#endif	/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000..36391d4
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,67 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/resovler_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _RESOLVER_INTERNAL_H
+#define _RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t	pid;	/* this resolver's PID, or 0 if not active */
+	Oid		dbid;	/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool	in_use;
+
+	/* Stats */
+	TimestampTz	last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t	mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	*latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/*
+	 * Foreign transaction resolution queues. Protected by FdwXactLock.
+	 */
+	SHM_QUEUE	FdwXactActiveQueue;
+	SHM_QUEUE	FdwXactRetryQueue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch		*launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif	/* _RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 0bbe9879..c15dff7 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDW_XACT_ID, "Foreign Transactions", fdw_xact_redo, fdw_xact_desc, fdw_xact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 0e932da..b199c88 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 				TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 2c1b2d8..63c833d 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -105,6 +105,13 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE				(1U << 3)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 30610b3..795e85a 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -227,6 +227,7 @@ typedef struct xl_parameter_change
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index 773d9e6..3d5333a 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -178,6 +178,7 @@ typedef struct ControlFileData
 	int			MaxConnections;
 	int			max_worker_processes;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 9264a2e..ee68caa 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5036,6 +5036,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '6053', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_fdwxact_resolver', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_fdwxact_resolver' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5741,6 +5748,22 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '6050', descr => 'view foreign transactions',
+  proname => 'pg_prepared_fdw_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,text}',
+  proargmodes => '{o,o,o,o,o,o}',
+  proargnames => '{dbid,transaction,serverid,userid,status,identifier}',
+  prosrc => 'pg_prepared_fdw_xacts' },
+{ oid => '6051', descr => 'remove foreign transaction',
+  proname => 'pg_remove_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_remove_fdw_xact' },
+{ oid => '6052', descr => 'resolve foreign transaction',
+  proname => 'pg_resolve_fdw_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  prosrc => 'pg_resolve_fdw_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index c14eb54..92d47bb 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/relation.h"
@@ -168,6 +169,14 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef bool (*PrepareForeignTransaction_function) (FdwXactState *state);
+typedef bool (*CommitForeignTransaction_function) (FdwXactState *state);
+typedef bool (*RollbackForeignTransaction_function) (FdwXactState *state);
+typedef bool (*ResolveForeignTransaction_function) (FdwXactState *state,
+													bool is_commit);
+typedef bool (*IsTwoPhaseCommitEnabled_function) (Oid serverid);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -235,6 +244,14 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for distributed transactions */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	ResolveForeignTransaction_function ResolveForeignTransaction;
+	IsTwoPhaseCommitEnabled_function IsTwoPhaseCommitEnabled;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
@@ -247,7 +264,6 @@ typedef struct FdwRoutine
 	ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild;
 } FdwRoutine;
 
-
 /* Functions in foreign/foreign.c */
 extern FdwRoutine *GetFdwRoutine(Oid fdwhandler);
 extern Oid	GetForeignServerIdByRelId(Oid relid);
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 3ca12e6..d030368 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -68,10 +68,10 @@ typedef struct ForeignTable
 	List	   *options;		/* ftoptions as DefElem list */
 } ForeignTable;
 
-
 extern ForeignServer *GetForeignServer(Oid serverid);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperByName(const char *name,
 							bool missing_ok);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index f1c10d1..05feb0a 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -759,6 +759,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDW_XACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDW_XACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -833,7 +835,8 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDW_XACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -913,6 +916,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_READ,
+	WAIT_EVENT_FDW_XACT_FILE_WRITE,
+	WAIT_EVENT_FDW_XACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index cb613c8..45880b2 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -153,6 +153,16 @@ struct PGPROC
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction
+								 * resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+
+	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
 	 * their lock.
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index 75bab29..25d6a2f 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDW_XACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -124,4 +126,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 								TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 668d9ef..81560bd 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -94,6 +94,8 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 735dd37..fdd6ded 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1413,6 +1413,13 @@ pg_policies| SELECT n.nspname AS schemaname,
    FROM ((pg_policy pol
      JOIN pg_class c ON ((c.oid = pol.polrelid)))
      LEFT JOIN pg_namespace n ON ((n.oid = c.relnamespace)));
+pg_prepared_fdw_xacts| SELECT f.dbid,
+    f.transaction,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.identifier
+   FROM pg_prepared_fdw_xacts() f(dbid, transaction, serverid, userid, status, identifier);
 pg_prepared_statements| SELECT p.name,
     p.statement,
     p.prepare_time,
@@ -1821,6 +1828,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_fdwxact_resolvers| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_fdwxact_resolver() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_progress_vacuum| SELECT s.pid,
     s.datid,
     d.datname,
-- 
2.10.5

From cdfdf20e785fbf44c1b12277862e41a43efbde14 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Fri, 8 Feb 2019 10:44:54 +0900
Subject: [PATCH v23 1/4] Keep track of writing on non-temporary relation

---
 src/backend/executor/nodeModifyTable.c | 12 ++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 18 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 8c0a2c4..e0c4e0a 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -582,6 +582,10 @@ ExecInsert(ModifyTableState *mtstate,
 						 estate->es_output_cid,
 						 0, NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -932,6 +936,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1441,6 +1449,10 @@ lreplace:;
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL, NIL);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	if (canSetTag)
 		(estate->es_processed)++;
 
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index d787f92..d6803d6 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -103,6 +103,12 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
 /*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
-- 
2.10.5

From 31b5cd4f4e7d9e7ff84721fdd0aab183de1189bb Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:48:08 +0900
Subject: [PATCH v23 4/4] Add regression tests for atomic commit.

---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index 648dd7e..b17429f 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000..9af9bb8
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 189abbb..0ab9f17 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2307,9 +2307,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2324,7 +2327,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.10.5

From 28979f2a4a6887ef687d3d95b615f62a4a42a2b4 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:46:01 +0900
Subject: [PATCH v23 3/4] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/Makefile                  |   7 +-
 contrib/postgres_fdw/connection.c              | 608 ++++++++++++++++---------
 contrib/postgres_fdw/expected/postgres_fdw.out | 261 ++++++++++-
 contrib/postgres_fdw/fdwxact.conf              |   3 +
 contrib/postgres_fdw/postgres_fdw.c            |  21 +-
 contrib/postgres_fdw/postgres_fdw.h            |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql      | 119 +++++
 doc/src/sgml/postgres-fdw.sgml                 |  46 ++
 8 files changed, 833 insertions(+), 239 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index 85394b4..5198f40 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -10,7 +10,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -23,3 +23,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index 239d220..40ddaa1 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2019, PostgreSQL Global Development Group
  *
@@ -14,9 +14,12 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
-#include "catalog/pg_user_mapping.h"
 #include "access/xact.h"
+#include "catalog/pg_user_mapping.h"
+#include "commands/defrem.h"
+#include "foreign/fdwapi.h"
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -56,6 +59,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -69,17 +73,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 					   SubTransactionId mySubid,
 					   SubTransactionId parentSubid,
@@ -89,26 +89,28 @@ static void pgfdw_reject_incomplete_xact_state_change(ConnCacheEntry *entry);
 static bool pgfdw_cancel_query(PGconn *conn);
 static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 						 bool ignore_errors);
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 						 PGresult **result);
-
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +130,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,12 +137,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -155,6 +150,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -182,6 +192,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +201,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +212,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,11 +228,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -414,7 +463,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -645,193 +694,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 }
 
 /*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow remote transactions that modified anything,
-					 * since it's not very reasonable to hold them open until
-					 * the prepared transaction is committed.  For the moment,
-					 * throw error unconditionally; later we might allow
-					 * read-only cases.  Note that the error will cause us to
-					 * come right back here with event == XACT_EVENT_ABORT, so
-					 * we'll clean up the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot prepare a transaction that modified remote tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
-/*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
 static void
@@ -847,10 +709,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -861,6 +719,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1195,3 +1057,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recusively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 3ed5270..4c787a0 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -179,15 +198,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8591,3 +8612,225 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
+ count 
+-------
+     0
+(1 row)
+
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000..3fdbf93
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index db62caf..648aeb8 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -499,7 +500,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 				  const PgFdwRelationInfo *fpinfo_o,
 				  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -553,6 +553,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1441,7 +1446,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2323,7 +2328,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2624,7 +2629,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3425,7 +3430,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4298,7 +4303,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4388,7 +4393,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4611,7 +4616,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index b382945..4b945d1 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/pathnodes.h"
@@ -121,7 +122,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -129,6 +130,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 				   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int ExtractConnectionOptions(List *defelems,
@@ -192,6 +196,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 						bool is_subquery,
 						List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 3bfcdab..f7954ce 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2419,3 +2442,99 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index a46fd75..5f8fe27 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -436,6 +436,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -464,6 +501,14 @@
   </para>
 
   <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
+  <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
    isolation level; otherwise it uses <literal>REPEATABLE READ</literal>
@@ -478,6 +523,7 @@
    COMMITTED</literal> local transaction.  A future
    <productname>PostgreSQL</productname> release might modify these rules.
   </para>
+
  </sect2>
 
  <sect2>
-- 
2.10.5

From de1a1c3530c6f79be1ded2cbc773517afecbd5dd Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:44:42 +0900
Subject: [PATCH v23 2/4] Support atomic commit among multiple foreign servers.

---
 doc/src/sgml/catalogs.sgml                    |  145 ++
 doc/src/sgml/config.sgml                      |  146 +-
 doc/src/sgml/distributed-transaction.sgml     |  158 ++
 doc/src/sgml/fdwhandler.sgml                  |  236 ++
 doc/src/sgml/filelist.sgml                    |    1 +
 doc/src/sgml/func.sgml                        |   89 +
 doc/src/sgml/monitoring.sgml                  |   60 +
 doc/src/sgml/postgres.sgml                    |    1 +
 doc/src/sgml/storage.sgml                     |    6 +
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  130 ++
 src/backend/access/fdwxact/fdwxact.c          | 2833 +++++++++++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  644 ++++++
 src/backend/access/fdwxact/resolver.c         |  344 +++
 src/backend/access/rmgrdesc/Makefile          |    8 +-
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   58 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/Makefile           |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/copy.c                   |    7 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    9 +
 src/backend/executor/nodeForeignscan.c        |   25 +
 src/backend/executor/nodeModifyTable.c        |   18 +
 src/backend/foreign/foreign.c                 |   57 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   20 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   82 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  165 ++
 src/include/access/fdwxact_launcher.h         |   29 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   54 +
 src/include/access/resolver_internal.h        |   66 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   29 +
 src/include/foreign/fdwapi.h                  |   13 +-
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    9 +-
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    3 +
 src/test/regress/expected/rules.out           |   13 +
 64 files changed, 5782 insertions(+), 27 deletions(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100755 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 1701863..4d6aa16 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8201,6 +8201,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
      </row>
 
      <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
+     <row>
       <entry><link linkend="view-pg-file-settings"><structname>pg_file_settings</structname></link></entry>
       <entry>summary of configuration file contents</entry>
      </row>
@@ -9640,6 +9645,146 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with that this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on that the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>initial</literal> : Initial status.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>resolved</literal> : This foreign transaction has been resolved.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-dbout status and
+       needs to be resolved by calling <function>pg_resolve_fdwxact</function>
+       function.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index ade44b5..867814f 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4281,7 +4281,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -8609,6 +8608,151 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign
+         transaction to be resolved before the command returns a "success"
+         indication to the client. Valid values are <literal>required</literal>,
+         <literal>prefer</literal> and <literal>disabled</literal>. The default
+         setting is <literal>disabled</literal>. Setting to
+         <literal>disabled</literal> don't use two-phase commit protocol to
+         commit or rollback distributed transactions. When set to
+         <literal>required</literal> the distributed transaction strictly
+         requires that all written servers can use two-phase commit protocol.
+         That is, the distributed transaction cannot commit if even one server
+         does not support the transaction management callback routines
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. Note that when <literal>disabled</literal> or
+         <literal>prefer</literal> there can be risk of database consistency
+         among all servers that involved in the distributed transaction when some
+         foreign server crashes during committing the distributed transaction.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero
+         value to set this parameter either <literal>required</literal> or
+         <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000..350b1af
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,158 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).
+    A <productname>PostgreSQL</productname> server that received SQL is called
+    <firstterm>coordinator node</firstterm> who is responsible for coordinating
+    all the partipanting transactions. Using two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed
+    transaction manager prepares all transaction on the foreign servers if
+    two-phase commit is required. Two-phase commit is required when the
+    transaction modifies data on two or more servers including the local server
+    itself and <xref linkend="guc-foreign-twophase-commit"/>is
+    <literal>required</literal> or <literal>prefer</literal>. If all preparations
+    on foreign servers got successful go to the next step. Any failure happens
+    in this step <productname>PostgreSQL</productname> changes to rollback, then
+    rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the
+    transaction locally. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes rollback, then rollback all
+    transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolution">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that is
+    responsible for foreign transaction resolution. They commit or rollback all
+    prepared transaction on foreign servers if the coordinator received agreement
+    messages from all foreign servers during the first step.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on one database of the coordinator side. On failure during resolution, they
+    retries to resolve at an interval of
+    <varname>foreign_transaction_resolution_interval</varname> time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped. So to drop the database, you can call
+     <function>pg_stop_foreign_xact_resovler</function> function before dropping
+     the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>Manual Resolution of In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, distributed transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the coordinator node crashed during either preparing or
+    resolving distributed transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="pg-stat-foreign-xact-view"/>
+    view. These foreign transactions need to be resolved by using
+    <function>pg_resolve_foriegn_xact</function> function.
+    <productname>PostgreSQL</productname> doesn't have facilities to automatially
+    resolve in-doubt transactions. These behavior might change in a future release.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-monitoring">
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-foreign-xact-view"><literal>pg_stat_foreign_xact</literal></link>
+    view. This view contains one row for every foreign transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 2c07a86..118e2a7 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1396,6 +1396,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1875,4 +1996,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 7e37042..c4300a1 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 6211ff3..7d39e7f 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -21895,6 +21895,95 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 547fe4c..9c71ea6 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -368,6 +368,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_foreign_xact</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1236,6 +1244,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
+        <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
         </row>
@@ -1459,6 +1479,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
+        <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
          <entry>Waiting during base backup when throttling activity.</entry>
@@ -2338,6 +2362,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-view" xreflabel="pg_stat_foreign_xact">
+   <title><structname>pg_stat_foreign_xact</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index 3e115f1..5ae3807 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -163,6 +163,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index e0915b6..89da268 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -84,6 +84,12 @@ Item
 </row>
 
 <row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
+<row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
 </row>
diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a..49480dd 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000..0207a66
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000..a6a46ad
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,130 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a conssitent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+--------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consisnts
+of the following four steps:
+
+1. Foriegn Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+we record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE all foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared tranascation on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step(commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itselft to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rolbacked the resolver process wake up the waiter.
+
+
+API Contract With Transaction Management Callback Functions
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForiegnTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for either PREPARE, COMMIT or
+ROLLBACK the trasaction on the foreign server respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transactio in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions has an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_INITIAL
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_PREPARING, FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING
+before the foreign transaction is prepared, committed and aborted by FDW
+callback functions respectively(*1). And the status then changes to
+FDWXACT_STATUS_RESOLVED once the foreign transaction are resolved, and then
+the corresponding FdwXact entry is removed with WAL logging. If failed during
+processing foreign transaction (i.g. preparing, committing or aborting) the
+status changes back to the previous status. Therefore the status
+FDWXACT_STATUS_xxxING appear only during the foreign transaction is being
+processed by an FDW callback function.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*2). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                      INVALID                       |
+ +----------------------------------------------------+
+    |                      |                       |
+    |                      v                       |
+    |           +---------------------+            |
+    |           |       INITIAL       |            |
+    |           +---------------------+            |
+   (*2)                    |                      (*2)
+    |                      v                       |
+    |           +---------------------+            |
+    |           |    PREPARING(*1)    |            |
+    |           +---------------------+            |
+    |                      |                       |
+    v                      v                       v
+ +----------------------------------------------------+
+ |                      PREPARED                      |
+ +----------------------------------------------------+
+           |                               |
+           v                               v
+ +--------------------+          +--------------------+
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |
+ +--------------------+          +--------------------+
+           |                               |
+           v                               v
+ +----------------------------------------------------+
+ |                      RESOLVED                      |
+ +----------------------------------------------------+
+
+(*1) Status that appear only during being processed by FDW
+(*2) Paths for recovered FdwXact entries
\ No newline at end of file
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100755
index 0000000..6a63663
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2833 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers automically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit. The foreign servers are
+ * registered if FDW has both CommitForeignTransaciton API and
+ * RollbackForeignTransactionAPI. Registered participant servers are identified
+ * by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request. We have one queue
+ * of which elements are ordered by the timestamp that they expect to be
+ * processed at. Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp that they expects to be processed.
+ * Similary if failed to resolve them, it enqueues again with new timestamp
+ * (its timestamp + foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction). Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by pg_resovle_fdwxact()
+ * function.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or
+ * more servers including itself. In other case, all foreign transactions are
+ * committed or rolled back during pre-commit.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed by FDW, the corresponding
+ * FdwXact entry is update. In order to protect the entry from concurrent
+ * removing we need to hold a lock on the entry or a lock for entire global
+ * array. However, we don't want to hold the lock during FDW is processing the
+ * foreign transaction that may take a unpredictable time. To avoid this, the
+ * in-memory data of foreign transaction follows a locking model based on
+ * four linked concepts:
+ *
+ * * A foreign transaction's status variable is switched using the LWLock
+ *   FdwXactLock, which need to be hold in exclusive mode when updating the
+ *   status, while readers need to hold it in shared mode when looking at the
+ *   status.
+ * * A process who is going to update FdwXact entry cannot process foreign
+ *   transaction that is being resolved.
+ * * So setting the status to FDWACT_STATUS_PREPARING,
+ *   FDWXACT_STATUS_COMMITTING or FDWXACT_STATUS_ABORTING, which makes foreign
+ *   transaction in-progress states, means to own the FdwXact entry, which
+ *   protect it from updating/removing by concurrent writers.
+ * * Individual fields are protected by mutex where only the backend owning
+ *   the foreign transaction is authorized to update the fields from its own
+ *   one.
+
+ * Therefore, before doing PREPARE, COMMIT PREPARED or ROLLBACK PREPARED a
+ * process who is going to call transaction callback functions needs to change
+ * the status to the corresponding status above while holding FdwXactLock in
+ * exclusive mode, and call callback function after releasing the lock.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk. FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon. We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts. If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to dink.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Atomic commit is enabled by configuration */
+#define IsForeignTwophaseCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	(IsForeignTwophaseCommitEnabled() && \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED))
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define IsSeverCapableOfTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/* Check the FdwXact is begin resolved */
+#define FdwXactIsBeingResolved(fx) \
+	(((((FdwXact)(fx))->status) == FDWXACT_STATUS_PREPARING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_COMMITTING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_ABORTING))
+
+/*
+ * Structure to bundle the foreign transaction participant. This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry
+	 * is not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char			*fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool			modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact_fn;
+	CommitForeignTransaction_function	commit_foreign_xact_fn;
+	RollbackForeignTransaction_function	rollback_foreign_xact_fn;
+	GetPrepareId_function				get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit. This list
+ * has only foreign servers that provides transaction management callbacks,
+ * that is CommitForeignTransaction and RollbackForeignTransaction.
+ */
+static List *FdwXactParticipants = NIL;
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+int foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid,
+										 FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+												 bool for_commit);
+static void FdwXactResolveForeignTransaction(FdwXact fdwxact,
+											 FdwXactRslvState *state,
+											 FdwXactStatus fallback_status);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid,	void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static void FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock);
+static bool is_foreign_twophase_commit_required(void);
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static List *get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						  bool including_indoubts, bool include_in_progress,
+						  bool need_lock);
+static FdwXact get_all_fdwxacts(int *num_p);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXact get_fdwxact_to_resolve(Oid dbid, TransactionId xid);
+static FdwXactRslvState *create_fdwxact_state(void);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation		rel;
+	Oid				serverid;
+	Oid				userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign transaction identified by given server id and user id.
+ * Registered foreign transactions are managed by the global transaction
+ * manager until the end of the transaction.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant	*fdw_part;
+	ForeignServer 		*foreign_server;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	MemoryContext		old_ctx;
+	FdwRoutine			*routine;
+	ListCell	   		*lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Don't register foreign server if it doesn't provide both commit and
+	 * rollback transaction management callbacks.
+	 */
+	if (!routine->CommitForeignTransaction ||
+		!routine->RollbackForeignTransaction)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		pfree(routine);
+		return;
+	}
+
+	/*
+	 * Remember we touched the foreign server that is not capable of two-phase
+	 * commit.
+	 */
+	if (!routine->PrepareForeignTransaction)
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+	foreign_server = GetForeignServer(serverid);
+	fdw = GetForeignDataWrapper(foreign_server->fdwid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&(fdwxacts[cnt].mutex));
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * If foreign twophase commit is required, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign server's FDWs to support two-phase commit protocol and ask them to
+ *  prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and for 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_twophase_commit;
+	ListCell	*lc = NULL;
+	ListCell	*next = NULL;
+	ListCell	*prev = NULL;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * we require all modified server have to be capable of two-phase
+	 * commit protocol.
+	 */
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false
+	 * if foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = is_foreign_twophase_commit_required();
+
+	/*
+	 * Firstly, we consider to commit foreign transactions in one-phase.
+	 */
+	for (lc = list_head(FdwXactParticipants); lc != NULL; lc = next)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool	commit = false;
+
+		next = lnext(lc);
+
+		/* Can commit in one-phase if two-phase commit is not requried */
+		if (!need_twophase_commit)
+			commit = true;
+
+		/* Non-modified foreign transaction always can be committed in one-phase */
+		if (!fdw_part->modified)
+			commit = true;
+
+		/*
+		 * In 'prefer' case, non-twophase-commit capable server can be
+		 * committed in one-phase.
+		 */
+		if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER &&
+			!IsSeverCapableOfTwophaseCommit(fdw_part))
+			commit = true;
+
+		if (commit)
+		{
+			/* Commit the foreign transaction in one-phase */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, true);
+
+			/* Delete it from the participant list */
+			FdwXactParticipants = list_delete_cell(FdwXactParticipants,
+												   lc, prev);
+			continue;
+		}
+
+		prev = lc;
+	}
+
+	/* All done if we committed all foreign transactions */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * Secondary, if only one transaction is remained in the participant list
+	 * and we didn't modified the local data we can commit it without
+	 * preparation.
+	 */
+	if (list_length(FdwXactParticipants) == 1 &&
+		(MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) == 0)
+	{
+		/* Commit the foreign transaction in one-phase */
+		FdwXactOnePhaseEndForeignTransaction(linitial(FdwXactParticipants),
+											 true);
+
+		/* All foreign transaction must be committed */
+		list_free(FdwXactParticipants);
+		return;
+	}
+
+	/*
+	 * Finally, prepare foreign transactions. Note that we keep
+	 * FdwXactParticipants until the end of transaction.
+	 */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here. If any error occurs, we rollback
+ * non-prepared foreign trasactions and leave others to the resolver.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell		*lcell;
+	TransactionId	xid;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	xid = GetTopTransactionId();
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXactRslvState 	*state;
+		FdwXact		fdwxact;
+
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, xid);
+
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the FDWXACT_STATUS_PREPARING
+		 * status. Registration persists this information to the disk and logs
+		 * (that way relaying it on standby). Thus in case we loose connectivity
+		 * to the foreign server or crash ourselves, we will remember that we
+		 * might have prepared transaction on the foreign server and try to
+		 * resolve it when connectivity is restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(xid, fdw_part);
+
+		state = create_fdwxact_state();
+		state->server = fdw_part->server;
+		state->usermapping = fdw_part->usermapping;
+		state->fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+
+		/* Update the status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		Assert(fdwxact->status == FDWXACT_STATUS_INITIAL);
+		fdwxact->status = FDWXACT_STATUS_PREPARING;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 *
+		 * During abort processing, we might try to resolve a never-preapred
+		 * transaction, and get an error. This is fine as long as the FDW
+		 * provides us unique prepared transaction identifiers.
+		 */
+		PG_TRY();
+		{
+			fdw_part->prepare_foreign_xact_fn(state);
+		}
+		PG_CATCH();
+		{
+			/* failed, back to the initial state */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			fdwxact->status = FDWXACT_STATUS_INITIAL;
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		/* succeeded, update status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * One-phase commit or rollback the given foreign transaction participant.
+ */
+static void
+FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+									 bool for_commit)
+{
+	FdwXactRslvState *state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state = create_fdwxact_state();
+	state->server = fdw_part->server;
+	state->usermapping = fdw_part->usermapping;
+	state->flags = FDWXACT_FLAG_ONEPHASE;
+
+	/*
+	 * Commit or rollback foreign transaction in one-phase. Since we didn't
+	 * insert FdwXact entry for this transaction we don't need to care
+	 * failures. On failure we change to rollback.
+	 */
+	if (for_commit)
+		fdw_part->commit_foreign_xact_fn(state);
+	else
+		fdw_part->rollback_foreign_xact_fn(state);
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fdwxact;
+	FdwXactOnDiskData	*fdwxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							fdw_part->usermapping->userid,
+							fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_INITIAL;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				Oid umid, char *fdwxact_id)
+{
+	int i;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->dbid = dbid;
+	fdwxact->local_xid = xid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	fdwxact->indoubt = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (FdwXactIsBeingResolved(fdwxact))
+		elog(ERROR, "cannot remove fdwxact entry that is beging resolved");
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true and set FdwXactAtomicCommitReady to true if the current transaction
+ * modified data on two or more servers in FdwXactParticipants and
+ * local server itself.
+ */
+static bool
+is_foreign_twophase_commit_required(void)
+{
+	ListCell*	lc;
+	int			nserverswritten = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	/*
+	 * Atomic commit is required if we modified data on two or more
+	 * participants.
+	 */
+	if (nserverswritten <= 1)
+		return false;
+
+	ForeignTwophaseCommitIsRequired = true;
+	return true;
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_lefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		if (!fdw_part->fdwxact)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in FdwXactParticipants
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget. So we need to check if the entries are still
+		 * associated with the transaction.
+		 */
+		SpinLockAcquire(&fdwxact->mutex);
+		if (fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+			fdwxact->indoubt = true;
+			n_lefts++;
+		}
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (n_lefts > 0)
+	{
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", n_lefts);
+		FdwXactComputeRequiredXmin();
+	}
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * Wait for the foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsForeignTwophaseCommitRequested())
+		return;
+
+	/*
+	 * Also, exit if the transaction itself has no foreign transaction
+	 * participants.
+	 */
+	if (FdwXactParticipants == NIL && wait_xid == MyPgXact->xid)
+		return;
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		foreach (lcell, FdwXactParticipants)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * If the foreign transaction has FdwXact entry we might have
+			 * prepared it. Skip already-prepared foreign transaction because
+			 * it has closed its transaction. But we are not sure that foreign
+			 * transaction with status == FDWXACT_STATUS_PREPARING has been
+			 * prepared or not. So we call the rollback API to close its
+			 * transaction for safety. The prepared foreign transaction that
+			 * we might have will be resolved by the foreign transaction
+			 * resolver.
+			 */
+			if (fdw_part->fdwxact)
+			{
+				bool is_prepared;
+
+				LWLockAcquire(FdwXactLock, LW_SHARED);
+				is_prepared = fdw_part->fdwxact &&
+					fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED;
+				LWLockRelease(FdwXactLock);
+
+				if (is_prepared)
+					continue;
+			}
+
+			/* One-phase rollback foreign transaction */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, false);
+		}
+	}
+
+	/*
+	 * In commit cases, we have already prepared foreign transactions during
+	 * pre-commit phase. And these prepared transactions will be resolved by
+	 * the resolver process.
+	 */
+
+	ForgetAllFdwXactParticipants();
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Prepare foreign transactions.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (is_foreign_twophase_commit_required() &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p)
+{
+	PGPROC *proc;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+			break;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+	{
+		*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+		*waitXid_p = proc->fdwXactWaitXid;
+	}
+	else
+	{
+		*nextResolutionTs_p = -1;
+		*waitXid_p = InvalidTransactionId;
+	}
+
+	LWLockRelease(FdwXactResolutionLock);
+
+	return proc;
+}
+
+/*
+ * Get one FdwXact entry to resolve. This function intended to be used when
+ * a resolver process get FdwXact entries to resolve. So we search entries
+ * while not including in-doubt transactions and in-progress transactions.
+ */
+static FdwXact
+get_fdwxact_to_resolve(Oid dbid, TransactionId xid)
+{
+	List *fdwxacts = NIL;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Don't include both in-doubt transactions and in-progress transactions */
+	fdwxacts = get_fdwxacts(dbid, xid, InvalidOid, InvalidOid,
+							false, false, false);
+
+	return fdwxacts == NIL ? NULL : (FdwXact) linitial(fdwxacts);
+}
+
+/*
+ * Resolve one distributed transaction on the given database . The target
+ * distributed transaction is fetched from the waiting queue and its transaction
+ * participants are fetched from the global array.
+ *
+ * Release the waiter and return true after we resolved the all of the foreign
+ * transaction participants. On failure, we re-enqueue the waiting backend after
+ * incremented the next resolution time.
+ */
+void
+FdwXactResolveTransactionAndReleaseWaiter(Oid dbid, TransactionId xid,
+										  PGPROC *waiter)
+{
+	FdwXact	fdwxact;
+
+	Assert(TransactionIdIsValid(xid));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	while ((fdwxact = get_fdwxact_to_resolve(MyDatabaseId, xid)) != NULL)
+	{
+		FdwXactRslvState *state;
+		ForeignServer *server;
+		UserMapping	*usermapping;
+
+		CHECK_FOR_INTERRUPTS();
+
+		server = GetForeignServer(fdwxact->serverid);
+		usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+
+		state = create_fdwxact_state();
+		SpinLockAcquire(&fdwxact->mutex);
+		state->server = server;
+		state->usermapping = usermapping;
+		state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+		SpinLockRelease(&fdwxact->mutex);
+
+		FdwXactDetermineTransactionFate(fdwxact, false);
+
+		/* Do not hold during foreign transaction resolution */
+		LWLockRelease(FdwXactLock);
+
+		PG_TRY();
+		{
+			/*
+			 * Resolve the foreign transaction. When committing or aborting
+			 * prepared foreign transactions the previous status is always
+			 * FDWXACT_STATUS_PREPARED.
+			 */
+			FdwXactResolveForeignTransaction(fdwxact, state,
+											 FDWXACT_STATUS_PREPARED);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDWXACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				waiter->fdwXactNextResolutionTs =
+					TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+												foreign_xact_resolution_retry_interval);
+				FdwXactQueueInsert(waiter);
+			}
+			LWLockRelease(FdwXactResolutionLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved one foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter
+	 * could already be detached if user cancelled to wait before
+	 * resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId	wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/* Wake up the waiter only when we have set state and removed from queue */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Determine whether the given foreign transaction should be committed or
+ * rolled back according to the result of the local transaction. This function
+ * changes fdwxact->status so the caller must hold FdwXactLock in exclusive
+ * mode or passing need_lock with true.
+ */
+static void
+FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock)
+{
+	bool			is_commit = false;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * The being resolved transaction must be either that has been cancelled
+	 *  and marked as in-doubt or that has been prepared.
+	 */
+	Assert(fdwxact->indoubt ||
+		   fdwxact->status == FDWXACT_STATUS_PREPARED);
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	if (TransactionIdDidCommit(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_COMMITTING;
+		is_commit = true;
+	}
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+		is_commit = false;
+	}
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+		is_commit = false;
+	}
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is
+	 * about to be committed or aborted. This should not happen except for one
+	 * case where the local transaction is prepared and this foreign transaction
+	 * is being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid),
+				 errhint("The local transaction with xid %u might be prepared",
+						 fdwxact->local_xid)));
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * callback function. The 'state' is passed to the callback function. The fate of
+ * foreign transaction must be determined. If foreign transaction is resolved
+ * successfully, remove the FdwXact entry from the shared memory and also
+ * remove the corresponding on-disk file. If failed, the status of FdwXact
+ * entry changes to 'fallback_status' before erroring out.
+ */
+static void
+FdwXactResolveForeignTransaction(FdwXact fdwxact, FdwXactRslvState *state,
+								 FdwXactStatus fallback_status)
+{
+	ForeignServer		*server;
+	ForeignDataWrapper	*fdw;
+	FdwRoutine			*fdw_routine;
+	bool				is_commit;
+
+	Assert(state != NULL);
+	Assert(state->server && state->usermapping && state->fdwxact_id);
+	Assert(fdwxact != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+		elog(ERROR, "cannot resolve foreign transaction whose fate is not determined");
+
+	is_commit = fdwxact->status == FDWXACT_STATUS_COMMITTING;
+	LWLockRelease(FdwXactLock);
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	PG_TRY();
+	{
+		if (is_commit)
+			fdw_routine->CommitForeignTransaction(state);
+		else
+			fdw_routine->RollbackForeignTransaction(state);
+	}
+	PG_CATCH();
+	{
+		/* Back to the fallback status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = fallback_status;
+		LWLockRelease(FdwXactLock);
+
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	elog(DEBUG1, "successfully %s the foreign transaction with xid %u db %u server %u user %u",
+		 is_commit ? "committed" : "rolled back",
+		 fdwxact->local_xid, fdwxact->dbid, fdwxact->serverid,
+		 fdwxact->userid);
+
+	fdwxact->status = FDWXACT_STATUS_RESOLVED;
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Return palloc'd and initialized FdwXactRslvState.
+ */
+static FdwXactRslvState *
+create_fdwxact_state(void)
+{
+	FdwXactRslvState *state;
+
+	state = palloc(sizeof(FdwXactRslvState));
+	state->server = NULL;
+	state->usermapping = NULL;
+	state->fdwxact_id = NULL;
+	state->flags = 0;
+
+	return state;
+}
+
+/*
+ * Return at least one FdwXact entry that matches to given argument,
+ * otherwise return NULL. All arguments must be valid values so that it can
+ * search exactly one (or none) entry. Note that this function intended to be
+ * used for modifying the returned FdwXact entry, so the caller must hold
+ * FdwXactLock in exclusive mode and it doesn't include the in-progress
+ * FdwXact entries.
+ */
+static FdwXact
+get_one_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	/* Include in-dbout transactions but don't include in-progress ones */
+	fdwxact_list = get_fdwxacts(dbid, xid, serverid, userid,
+								true, false, false);
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdwxact_list) <= 1);
+
+	/* Could not find entry */
+	if (fdwxact_list == NIL)
+		return NULL;
+
+	return (FdwXact) linitial(fdwxact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdwxact_exists(Oid dbid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	/* Find entries from all FdwXact entries */
+	fdwxact_list = get_fdwxacts(dbid, InvalidTransactionId, serverid,
+								userid, true, true, true);
+
+	return fdwxact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_foreign_xacts, and the number of entries to num_p.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't
+ * want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdwxacts(int *num_p)
+{
+	List		*all_fdwxacts;
+	ListCell	*lc;
+	FdwXact		fdwxacts;
+	int			num_fdwxacts = 0;
+
+	Assert(num_p != NULL);
+
+	/* Get all entries */
+	all_fdwxacts = get_fdwxacts(InvalidOid, InvalidTransactionId,
+								InvalidOid, InvalidOid, true,
+								true, true);
+
+	if (all_fdwxacts == NIL)
+	{
+		*num_p = 0;
+		return NULL;
+	}
+
+	fdwxacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdwxacts));
+	*num_p = list_length(all_fdwxacts);
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdwxacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdwxacts + num_fdwxacts, fx,
+			   sizeof(FdwXactData));
+		num_fdwxacts++;
+	}
+
+	list_free(all_fdwxacts);
+
+	return fdwxacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respecitively.
+ */
+static List*
+get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			 bool include_indoubt, bool include_in_progress, bool need_lock)
+{
+	int i;
+	List	*fdwxact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact	fdwxact = FdwXactCtl->fdwxacts[i];
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* include in-doubt transaction? */
+		if (!include_indoubt && fdwxact->indoubt)
+			continue;
+
+		/* include in-progress transaction? */
+		if (!include_in_progress && FdwXactIsBeingResolved(fdwxact))
+			continue;
+
+		/* Append it if matched */
+		fdwxact_list = lappend(fdwxact_list, fdwxact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdwxact_list;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	*id;
+	int		id_len = 0;
+
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		/*
+		 * FDW doesn't provide the callback function, generate an unique
+		 * idenetifier.
+		 */
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+			 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifer \"%s\" is too long",
+						id),
+				 errdetail("foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;						/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdwxacts,
+							 serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid =
+		XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid  != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							  fdwxact_data->serverid, fdwxact_data->userid,
+							  fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know
+	 * the xact status right now. Resolver will set it later based on
+	 * the status of local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;	/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdwxact(dbid, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		return;
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(int *newval, void **extra, GucSource source)
+{
+	ForeignTwophaseCommitLevel newForeignTwophaseCommitLevel = *newval;
+
+	/* Parameter check */
+	if (newForeignTwophaseCommitLevel > FOREIGN_TWOPHASE_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	7
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdwxacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdwxacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(PG_PREPARED_FDWXACTS_COLS);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "indoubt",
+						   BOOLOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 7, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdwxacts = get_all_fdwxacts(&num_fdwxacts);
+		status->num_xacts = num_fdwxacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdwxact = &status->fdwxacts[status->cur_xact++];
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdwxact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdwxact->dbid);
+		values[1] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[2] = ObjectIdGetDatum(fdwxact->serverid);
+		values[3] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (fdwxact->status)
+		{
+			case FDWXACT_STATUS_INITIAL:
+				xact_status = "initial";
+				break;
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			case FDWXACT_STATUS_RESOLVED:
+				xact_status = "resolved";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		values[5] = BoolGetDatum(fdwxact->indoubt);
+		values[6] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	FdwXact			fdwxact;
+	FdwXactRslvState	*state;
+	FdwXactStatus		prev_status;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	server = GetForeignServer(serverid);
+	usermapping = GetUserMapping(userid, serverid);
+	state = create_fdwxact_state();
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+	{
+		LWLockRelease(FdwXactLock);
+		PG_RETURN_BOOL(false);
+	}
+
+	state->server = server;
+	state->usermapping = usermapping;
+	state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+
+	SpinLockAcquire(&fdwxact->mutex);
+	prev_status = fdwxact->status;
+	SpinLockRelease(&fdwxact->mutex);
+
+	FdwXactDetermineTransactionFate(fdwxact, false);
+
+	LWLockRelease(FdwXactLock);
+
+	FdwXactResolveForeignTransaction(fdwxact, state, prev_status);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000..45fb530
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,644 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to retry resolution.
+ */
+void
+FdwXactLauncherRequestToLaunchForRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			resolver->last_resolved_time = 0;
+			resolver->latch = NULL;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a foreign_xact_resolution_retry_interval
+		 * but always starts when the backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find FdwXact
+		 * entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int unused_slot;
+	int i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (unused_slot > max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	*resolver_dbs;	/* DBs resolver's running on */
+	HTAB	*fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL	ctl;
+	HASH_SEQ_STATUS status;
+	Oid		*entry;
+	bool	launched;
+	int		i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->indoubt)
+			continue;
+
+		hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+		return false;
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign trasaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		 /* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+						WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+						10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Returns activity of all foreign transaction resolvers.
+ */
+Datum
+pg_stat_get_foreign_xact(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000..9298877
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,344 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+bool foreign_xact_resolve_indoubt_xacts;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	MyFdwXactResolver->last_resolved_time = 0;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		PGPROC			*waiter = NULL;
+		TransactionId	waitXid = InvalidTransactionId;
+		TimestampTz		resolutionTs = -1;
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or got the waiter
+		 * that has future resolution time.
+		 */
+		while ((waiter = FdwXactGetWaiter(&resolutionTs, &waitXid)) != NULL)
+		{
+			CHECK_FOR_INTERRUPTS();
+			Assert(TransactionIdIsValid(waitXid));
+
+			if	(resolutionTs > now)
+				break;
+
+			elog(DEBUG2, "resolver got one waiter with xid %u", waitXid);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveTransactionAndReleaseWaiter(MyDatabaseId, waitXid,
+													  waiter);
+			CommitTransactionCommand();
+
+			/* Update my stats */
+			SpinLockAcquire(&(MyFdwXactResolver->mutex));
+			MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+			SpinLockRelease(&(MyFdwXactResolver->mutex));
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz last_resolved_time;
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	last_resolved_time = MyFdwXactResolver->last_resolved_time;
+	timeout = TimestampTzPlusMilliseconds(last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * checked FdwXactWaiterExists.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long	sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index 5514db1..742e825 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -8,9 +8,9 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
-	   gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
-	   mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
-	   smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdwxactdesc.o \
+	genericdesc.o  gindesc.o gistdesc.o hashdesc.o heapdesc.o \
+	logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \
+	seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000..fe0cef9
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxactdesc.c
+ *		PostgreSQL global transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdwxactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdwxact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		FdwXactOnDiskData *fdwxact_insert = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_insert->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_insert->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_insert->dbid);
+		appendStringInfo(buf, " local xid: %u,", fdwxact_insert->local_xid);
+		appendStringInfo(buf, " id: %s", fdwxact_insert->fdwxact_id);
+	}
+	else
+	{
+		xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_remove->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_remove->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_remove->dbid);
+		appendStringInfo(buf, " local xid: %u", fdwxact_remove->xid);
+	}
+
+}
+
+const char *
+fdwxact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDWXACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDWXACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 33060f3..1d4e1c8 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47..f15c83a 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -12,9 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
-	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
-	xact.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o generic_xlog.o multixact.o \
+	parallel.o rmgr.o slru.o subtrans.o timeline.o transam.o twophase.o \
+	twophase_rmgr.o varsup.o xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9368b56..8b360b1 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -9,6 +9,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
 #include "access/generic_xlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f9a4960..c625b45 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -852,6 +853,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 }
 
 /*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
+/*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
  *
@@ -2332,6 +2362,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2391,6 +2427,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 8522a2f..0b35cdb 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1217,6 +1218,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1225,6 +1227,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1263,12 +1266,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1426,6 +1430,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2085,6 +2097,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2244,6 +2259,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2331,6 +2347,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2525,6 +2543,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2736,6 +2755,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index c00b63c..61d4f3d 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
 #include "access/subtrans.h"
@@ -5250,6 +5251,7 @@ BootStrapXLOG(void)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6168,6 +6170,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -6690,14 +6695,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -6889,7 +6895,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7385,6 +7394,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7703,6 +7713,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -8978,6 +8991,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9411,8 +9425,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9430,6 +9446,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9446,6 +9463,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9651,6 +9669,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -9850,6 +9869,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 161bad6..b285fb0 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -291,6 +291,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
@@ -774,6 +777,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_foreign_xact AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_foreign_xact() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index c39218f..d77d35a 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2855,9 +2855,16 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
 
+	}
+
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
 
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index 413ce3f..5588681 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1105,6 +1107,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1422,6 +1436,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
 	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
+	/*
 	 * Do the deletion
 	 */
 	object.classId = UserMappingRelationId;
@@ -1574,6 +1597,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index 70709e5..b8b76d9 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,7 +13,9 @@
  */
 #include "postgres.h"
 
+
 #include "access/table.h"
+#include "access/fdwxact.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
 #include "catalog/pg_inherits.h"
@@ -944,7 +946,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 52af1da..eb7450c 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,10 +226,33 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
 
+	}
+
 	return scanstate;
 }
 
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index e0c4e0a..e9773a3 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -38,6 +38,7 @@
 #include "postgres.h"
 
 #include "access/heapam.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
 #include "access/xact.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -544,6 +546,10 @@ ExecInsert(ModifyTableState *mtstate,
 									 NULL,
 									 specToken);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -772,6 +778,10 @@ ldelete:;
 							  &tmfd,
 							  changingPart);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -1317,6 +1327,10 @@ lreplace:;
 							  true /* wait for commit */ ,
 							  &tmfd, &lockmode, &update_indexes);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -2375,6 +2389,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index c917ec4..2780ed5 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMapping - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,20 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction &&
+		 !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction &&
+		 routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data-wrapper must support both commit and rollback routine or either");
+
+	if (routine->PrepareForeignTransaction &&
+		(!routine->CommitForeignTransaction ||
+		 !routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index f5db5a8..10393d3 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -15,6 +15,8 @@
 #include <unistd.h>
 
 #include "libpq/pqsignal.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index cdf87ba..66cc834 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3574,6 +3574,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3777,6 +3783,11 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDWXACT:
+			event_name = "FdwXact";
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -3992,6 +4003,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 60d29a2..4d370f6 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -900,6 +902,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -975,12 +981,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index eec3a22..dc02e42 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -154,6 +154,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index d7d7335..1491bc6 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(int port)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(int port)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 010cc06..67ffcdb 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -91,6 +91,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -246,6 +248,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1313,6 +1316,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1378,6 +1382,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1428,6 +1433,15 @@ GetOldestXmin(Relation rel, int flags)
 		result = replication_slot_xmin;
 
 	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
+	/*
 	 * After locks have been released and defer_cleanup_age has been applied,
 	 * check whether we need to back up further to make logical decoding
 	 * possible. We need to do so if we're computing the global limit (rel =
@@ -3016,6 +3030,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index db47843..0b8a487 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -49,3 +49,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 CLogTruncationLock					44
+FdwXactLock					45
+FdwXactResolverLock			46
+FdwXactResolutionLock			47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 0da5b19..d57daf1 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -422,6 +423,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -823,6 +828,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 44a59e1..8b8330b 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3000,6 +3002,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index f7f726b..ace89d0 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -397,6 +398,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 };
 
 /*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
+/*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
  */
@@ -720,6 +740,12 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
+	/* FDWXACT_RESOLVER */
+	gettext_noop("Foreign Transaction Management / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2352,6 +2378,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FDWXACT_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FDWXACT_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4374,6 +4446,16 @@ static struct config_enum ConfigureNamesEnum[] =
 	},
 
 	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
+	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
 			NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 77bb7c2..ae212c7 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -341,6 +343,20 @@
 
 
 #------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
+#------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index 33ac627..328b857 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 09b59c8..99d4d39 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -210,6 +210,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index a674f52..3f666cc 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -310,6 +310,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 82a8ec9..4a3e524 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -717,6 +717,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -933,6 +934,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca..b616cea 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000..147d41c
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,165 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL global transaction manager
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDWXACT_H
+#define FDWXACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* fdwXactState */
+#define	FDWXACT_NOT_WAITING		0
+#define	FDWXACT_WAITING			1
+#define	FDWXACT_WAIT_COMPLETE	2
+
+/* Flag passed to FDW transaction management APIs */
+#define FDWXACT_FLAG_ONEPHASE		0x01	/* transaction can commit/rollback
+											   without preparation */
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER,		/* use twophase commit where available */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support
+										   twophase commit */
+} ForeignTwophaseCommitLevel;
+
+/* Enum to track the status of foreign transaction */
+typedef enum
+{
+	FDWXACT_STATUS_INVALID,
+	FDWXACT_STATUS_INITIAL,
+	FDWXACT_STATUS_PREPARING,		/* foreign transaction is being prepared */
+	FDWXACT_STATUS_PREPARED,		/* foreign transaction is prepared */
+	FDWXACT_STATUS_COMMITTING,		/* foreign prepared transaction is to
+									 * be committed */
+	FDWXACT_STATUS_ABORTING,		/* foreign prepared transaction is to be
+									 * aborted */
+	FDWXACT_STATUS_RESOLVED
+} FdwXactStatus;
+
+typedef struct FdwXactData *FdwXact;
+
+/*
+ * Shared memory state of a single foreign transaction.
+ */
+typedef struct FdwXactData
+{
+	FdwXact			fdwxact_free_next;	/* Next free FdwXact entry */
+
+	Oid				dbid;			/* database oid where to find foreign server
+									 * and user mapping */
+	TransactionId	local_xid;		/* XID of local transaction */
+	Oid				serverid;		/* foreign server where transaction takes
+									 * place */
+	Oid				userid;			/* user who initiated the foreign
+									 * transaction */
+	Oid				umid;
+	bool			indoubt;		/* Is an in-doubt transaction? */
+	slock_t			mutex;			/* Protect the above fields */
+
+	/* The status of the foreign transaction, protected by FdwXactLock */
+	FdwXactStatus 	status;
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;		/* XLOG offset of inserting this entry start */
+	XLogRecPtr	insert_end_lsn;		/* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN];		/* prepared transaction identifier */
+} FdwXactData;
+
+/*
+ * Shared memory layout for maintaining foreign prepared transaction entries.
+ * Adding or removing FdwXact entry needs to hold FdwXactLock in exclusive mode,
+ * and iterating fdwXacts needs that in shared mode.
+ */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		free_fdwxacts;
+
+	/* Number of valid foreign transaction entries */
+	int			num_fdwxacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdwxacts[FLEXIBLE_ARRAY_MEMBER];		/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* State data for foreign transaction resolution, passed to FDW callbacks */
+typedef struct FdwXactRslvState
+{
+	/* Foreign transaction information */
+	char	*fdwxact_id;
+
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	int		flags;			/* OR of FDWXACT_FLAG_xx flags */
+} FdwXactRslvState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+/* Function declarations */
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern bool fdwxact_exists(Oid dboid, Oid serverid, Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern void PreCommit_FdwXacts(void);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+extern void FdwXactResolveTransactionAndReleaseWaiter(Oid dbid, TransactionId xid,
+													  PGPROC *waiter);
+extern bool FdwXactResolveInDoubtTransactions(Oid dbid);
+extern PGPROC *FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p);
+extern void FdwXactCleanupAtProcExit(void);
+extern void RegisterFdwXactByRelId(Oid relid, bool modified);
+extern void RegisterFdwXactByServerId(Oid serverid, bool modified);
+extern void FdwXactMarkForeignServerAccessed(Oid relid, bool modified);
+extern bool check_foreign_twophase_commit(int *newval, void **extra,
+										  GucSource source);
+extern bool FdwXactWaiterExists(Oid dbid);
+
+#endif   /* FDWXACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000..dd0f5d1
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,29 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherRequestToLaunch(void);
+extern void FdwXactLauncherRequestToLaunchForRetry(void);
+extern void FdwXactLaunchOrWakeupResolver(void);
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+
+
+#endif	/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000..2607654
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int foreign_xact_resolver_timeout;
+
+#endif		/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000..39ca66b
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,54 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDWXACT_INSERT	0x00
+#define XLOG_FDWXACT_REMOVE	0x10
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDWXACT_ID_MAX_LEN 200
+
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdwxact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+	bool		force;
+} xl_fdwxact_remove;
+
+extern void fdwxact_redo(XLogReaderState *record);
+extern void fdwxact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdwxact_identify(uint8 info);
+
+#endif	/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000..55fc970
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,66 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/resovler_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t	pid;	/* this resolver's PID, or 0 if not active */
+	Oid		dbid;	/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool	in_use;
+
+	/* Stats */
+	TimestampTz	last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t	mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	*latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/* Foreign transaction resolution queue. Protected by FdwXactLock */
+	SHM_QUEUE	fdwxact_queue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch		*launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+#define SizeOfFdwXactRslvCtlData \
+	(offsetof(FdwXactRslvCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif	/* RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 3c0db2c..5798b4c 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Foreign Transactions", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index fcd1913..34dfa81 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 				TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index d6803d6..4e88b74 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -109,6 +109,13 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 8b1348c..f29b1b1 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -228,6 +228,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index ff98d9e..773846d 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 73ebfdf..01fc53b 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5153,6 +5153,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '6053', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_foreign_xact', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_foreign_xact' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5866,6 +5873,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '6050', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o,o}',
+  proargnames => '{dbid,xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '6051', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '6052', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -5984,6 +6009,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '6054',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index fd91eb6..a69b700 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
@@ -248,7 +260,6 @@ typedef struct FdwRoutine
 	ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild;
 } FdwRoutine;
 
-
 /* Functions in foreign/foreign.c */
 extern FdwRoutine *GetFdwRoutine(Oid fdwhandler);
 extern Oid	GetForeignServerIdByRelId(Oid relid);
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 64919c9..b82cb45 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -80,6 +80,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 						 bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 							  bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index fa5dca3..9aa84cb 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -775,6 +775,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -852,7 +854,9 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDWXACT,
+	WAIT_EVENT_FDWXACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -932,6 +936,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index 1cee7db..f9c4eb7 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -21,6 +21,7 @@
 #include "storage/lock.h"
 #include "storage/pg_sema.h"
 #include "storage/proclist_types.h"
+#include "datatype/timestamp.h"
 
 /*
  * Each backend advertises up to PGPROC_MAX_CACHED_SUBXIDS TransactionIds
@@ -153,6 +154,16 @@ struct PGPROC
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
+	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
 	 * their lock.
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index bd24850..d569fc0 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -124,4 +126,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 								TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 2a74b30..c530634 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,9 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
+	FDWXACT_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 3097390..492b254 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1341,6 +1341,14 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.dbid,
+    f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(dbid, xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
@@ -1841,6 +1849,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_foreign_xact| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_foreign_xact() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_gssapi| SELECT s.pid,
     s.gss_auth AS gss_authenticated,
     s.gss_princ AS principal,
-- 
2.10.5

From 28bb614ac2d7a82a080e178d7f37de47142b299d Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Fri, 8 Feb 2019 10:44:54 +0900
Subject: [PATCH v24 1/5] Keep track of writing on non-temporary relation

---
 src/backend/executor/nodeModifyTable.c | 12 ++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 18 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index d8b695d..a70549d 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -588,6 +588,10 @@ ExecInsert(ModifyTableState *mtstate,
 							   estate->es_output_cid,
 							   0, NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -940,6 +944,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1451,6 +1459,10 @@ lreplace:;
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL, NIL);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	if (canSetTag)
 		(estate->es_processed)++;
 
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index a20726a..c9d6b47 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -103,6 +103,12 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
 /*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
-- 
2.10.5

From 283f0a009ca26a12f0c6738ad4d6325d895f3b55 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Tue, 2 Jul 2019 09:32:16 +0900
Subject: [PATCH v24 2/5] Support atomic commit among multiple foreign servers.

---
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  130 ++
 src/backend/access/fdwxact/fdwxact.c          | 2833 +++++++++++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  644 ++++++
 src/backend/access/fdwxact/resolver.c         |  344 +++
 src/backend/access/rmgrdesc/Makefile          |    8 +-
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/Makefile           |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/copy.c                   |    7 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    9 +
 src/backend/executor/nodeForeignscan.c        |   25 +
 src/backend/executor/nodeModifyTable.c        |   18 +
 src/backend/foreign/foreign.c                 |   57 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   20 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   82 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   29 +
 src/include/foreign/fdwapi.h                  |   13 +-
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    9 +-
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    3 +
 src/test/regress/expected/rules.out           |   13 +
 49 files changed, 4546 insertions(+), 26 deletions(-)
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100755 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c

diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a..49480dd 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000..0207a66
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000..a6a46ad
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,130 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a conssitent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+--------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consisnts
+of the following four steps:
+
+1. Foriegn Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+we record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE all foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared tranascation on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step(commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itselft to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rolbacked the resolver process wake up the waiter.
+
+
+API Contract With Transaction Management Callback Functions
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForiegnTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for either PREPARE, COMMIT or
+ROLLBACK the trasaction on the foreign server respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transactio in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions has an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_INITIAL
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_PREPARING, FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING
+before the foreign transaction is prepared, committed and aborted by FDW
+callback functions respectively(*1). And the status then changes to
+FDWXACT_STATUS_RESOLVED once the foreign transaction are resolved, and then
+the corresponding FdwXact entry is removed with WAL logging. If failed during
+processing foreign transaction (i.g. preparing, committing or aborting) the
+status changes back to the previous status. Therefore the status
+FDWXACT_STATUS_xxxING appear only during the foreign transaction is being
+processed by an FDW callback function.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*2). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                      INVALID                       |
+ +----------------------------------------------------+
+    |                      |                       |
+    |                      v                       |
+    |           +---------------------+            |
+    |           |       INITIAL       |            |
+    |           +---------------------+            |
+   (*2)                    |                      (*2)
+    |                      v                       |
+    |           +---------------------+            |
+    |           |    PREPARING(*1)    |            |
+    |           +---------------------+            |
+    |                      |                       |
+    v                      v                       v
+ +----------------------------------------------------+
+ |                      PREPARED                      |
+ +----------------------------------------------------+
+           |                               |
+           v                               v
+ +--------------------+          +--------------------+
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |
+ +--------------------+          +--------------------+
+           |                               |
+           v                               v
+ +----------------------------------------------------+
+ |                      RESOLVED                      |
+ +----------------------------------------------------+
+
+(*1) Status that appear only during being processed by FDW
+(*2) Paths for recovered FdwXact entries
\ No newline at end of file
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100755
index 0000000..6a63663
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2833 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers automically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit. The foreign servers are
+ * registered if FDW has both CommitForeignTransaciton API and
+ * RollbackForeignTransactionAPI. Registered participant servers are identified
+ * by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request. We have one queue
+ * of which elements are ordered by the timestamp that they expect to be
+ * processed at. Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp that they expects to be processed.
+ * Similary if failed to resolve them, it enqueues again with new timestamp
+ * (its timestamp + foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction). Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by pg_resovle_fdwxact()
+ * function.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or
+ * more servers including itself. In other case, all foreign transactions are
+ * committed or rolled back during pre-commit.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed by FDW, the corresponding
+ * FdwXact entry is update. In order to protect the entry from concurrent
+ * removing we need to hold a lock on the entry or a lock for entire global
+ * array. However, we don't want to hold the lock during FDW is processing the
+ * foreign transaction that may take a unpredictable time. To avoid this, the
+ * in-memory data of foreign transaction follows a locking model based on
+ * four linked concepts:
+ *
+ * * A foreign transaction's status variable is switched using the LWLock
+ *   FdwXactLock, which need to be hold in exclusive mode when updating the
+ *   status, while readers need to hold it in shared mode when looking at the
+ *   status.
+ * * A process who is going to update FdwXact entry cannot process foreign
+ *   transaction that is being resolved.
+ * * So setting the status to FDWACT_STATUS_PREPARING,
+ *   FDWXACT_STATUS_COMMITTING or FDWXACT_STATUS_ABORTING, which makes foreign
+ *   transaction in-progress states, means to own the FdwXact entry, which
+ *   protect it from updating/removing by concurrent writers.
+ * * Individual fields are protected by mutex where only the backend owning
+ *   the foreign transaction is authorized to update the fields from its own
+ *   one.
+
+ * Therefore, before doing PREPARE, COMMIT PREPARED or ROLLBACK PREPARED a
+ * process who is going to call transaction callback functions needs to change
+ * the status to the corresponding status above while holding FdwXactLock in
+ * exclusive mode, and call callback function after releasing the lock.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk. FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon. We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts. If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to dink.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Atomic commit is enabled by configuration */
+#define IsForeignTwophaseCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	(IsForeignTwophaseCommitEnabled() && \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED))
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define IsSeverCapableOfTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/* Check the FdwXact is begin resolved */
+#define FdwXactIsBeingResolved(fx) \
+	(((((FdwXact)(fx))->status) == FDWXACT_STATUS_PREPARING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_COMMITTING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_ABORTING))
+
+/*
+ * Structure to bundle the foreign transaction participant. This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry
+	 * is not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char			*fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool			modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact_fn;
+	CommitForeignTransaction_function	commit_foreign_xact_fn;
+	RollbackForeignTransaction_function	rollback_foreign_xact_fn;
+	GetPrepareId_function				get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit. This list
+ * has only foreign servers that provides transaction management callbacks,
+ * that is CommitForeignTransaction and RollbackForeignTransaction.
+ */
+static List *FdwXactParticipants = NIL;
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+int foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid,
+										 FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+												 bool for_commit);
+static void FdwXactResolveForeignTransaction(FdwXact fdwxact,
+											 FdwXactRslvState *state,
+											 FdwXactStatus fallback_status);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid,	void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static void FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock);
+static bool is_foreign_twophase_commit_required(void);
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static List *get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						  bool including_indoubts, bool include_in_progress,
+						  bool need_lock);
+static FdwXact get_all_fdwxacts(int *num_p);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXact get_fdwxact_to_resolve(Oid dbid, TransactionId xid);
+static FdwXactRslvState *create_fdwxact_state(void);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation		rel;
+	Oid				serverid;
+	Oid				userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign transaction identified by given server id and user id.
+ * Registered foreign transactions are managed by the global transaction
+ * manager until the end of the transaction.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant	*fdw_part;
+	ForeignServer 		*foreign_server;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	MemoryContext		old_ctx;
+	FdwRoutine			*routine;
+	ListCell	   		*lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Don't register foreign server if it doesn't provide both commit and
+	 * rollback transaction management callbacks.
+	 */
+	if (!routine->CommitForeignTransaction ||
+		!routine->RollbackForeignTransaction)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		pfree(routine);
+		return;
+	}
+
+	/*
+	 * Remember we touched the foreign server that is not capable of two-phase
+	 * commit.
+	 */
+	if (!routine->PrepareForeignTransaction)
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+	foreign_server = GetForeignServer(serverid);
+	fdw = GetForeignDataWrapper(foreign_server->fdwid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&(fdwxacts[cnt].mutex));
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * If foreign twophase commit is required, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign server's FDWs to support two-phase commit protocol and ask them to
+ *  prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and for 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_twophase_commit;
+	ListCell	*lc = NULL;
+	ListCell	*next = NULL;
+	ListCell	*prev = NULL;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * we require all modified server have to be capable of two-phase
+	 * commit protocol.
+	 */
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false
+	 * if foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = is_foreign_twophase_commit_required();
+
+	/*
+	 * Firstly, we consider to commit foreign transactions in one-phase.
+	 */
+	for (lc = list_head(FdwXactParticipants); lc != NULL; lc = next)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool	commit = false;
+
+		next = lnext(lc);
+
+		/* Can commit in one-phase if two-phase commit is not requried */
+		if (!need_twophase_commit)
+			commit = true;
+
+		/* Non-modified foreign transaction always can be committed in one-phase */
+		if (!fdw_part->modified)
+			commit = true;
+
+		/*
+		 * In 'prefer' case, non-twophase-commit capable server can be
+		 * committed in one-phase.
+		 */
+		if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER &&
+			!IsSeverCapableOfTwophaseCommit(fdw_part))
+			commit = true;
+
+		if (commit)
+		{
+			/* Commit the foreign transaction in one-phase */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, true);
+
+			/* Delete it from the participant list */
+			FdwXactParticipants = list_delete_cell(FdwXactParticipants,
+												   lc, prev);
+			continue;
+		}
+
+		prev = lc;
+	}
+
+	/* All done if we committed all foreign transactions */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * Secondary, if only one transaction is remained in the participant list
+	 * and we didn't modified the local data we can commit it without
+	 * preparation.
+	 */
+	if (list_length(FdwXactParticipants) == 1 &&
+		(MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) == 0)
+	{
+		/* Commit the foreign transaction in one-phase */
+		FdwXactOnePhaseEndForeignTransaction(linitial(FdwXactParticipants),
+											 true);
+
+		/* All foreign transaction must be committed */
+		list_free(FdwXactParticipants);
+		return;
+	}
+
+	/*
+	 * Finally, prepare foreign transactions. Note that we keep
+	 * FdwXactParticipants until the end of transaction.
+	 */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here. If any error occurs, we rollback
+ * non-prepared foreign trasactions and leave others to the resolver.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell		*lcell;
+	TransactionId	xid;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	xid = GetTopTransactionId();
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXactRslvState 	*state;
+		FdwXact		fdwxact;
+
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, xid);
+
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the FDWXACT_STATUS_PREPARING
+		 * status. Registration persists this information to the disk and logs
+		 * (that way relaying it on standby). Thus in case we loose connectivity
+		 * to the foreign server or crash ourselves, we will remember that we
+		 * might have prepared transaction on the foreign server and try to
+		 * resolve it when connectivity is restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(xid, fdw_part);
+
+		state = create_fdwxact_state();
+		state->server = fdw_part->server;
+		state->usermapping = fdw_part->usermapping;
+		state->fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+
+		/* Update the status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		Assert(fdwxact->status == FDWXACT_STATUS_INITIAL);
+		fdwxact->status = FDWXACT_STATUS_PREPARING;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 *
+		 * During abort processing, we might try to resolve a never-preapred
+		 * transaction, and get an error. This is fine as long as the FDW
+		 * provides us unique prepared transaction identifiers.
+		 */
+		PG_TRY();
+		{
+			fdw_part->prepare_foreign_xact_fn(state);
+		}
+		PG_CATCH();
+		{
+			/* failed, back to the initial state */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			fdwxact->status = FDWXACT_STATUS_INITIAL;
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		/* succeeded, update status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * One-phase commit or rollback the given foreign transaction participant.
+ */
+static void
+FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+									 bool for_commit)
+{
+	FdwXactRslvState *state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state = create_fdwxact_state();
+	state->server = fdw_part->server;
+	state->usermapping = fdw_part->usermapping;
+	state->flags = FDWXACT_FLAG_ONEPHASE;
+
+	/*
+	 * Commit or rollback foreign transaction in one-phase. Since we didn't
+	 * insert FdwXact entry for this transaction we don't need to care
+	 * failures. On failure we change to rollback.
+	 */
+	if (for_commit)
+		fdw_part->commit_foreign_xact_fn(state);
+	else
+		fdw_part->rollback_foreign_xact_fn(state);
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fdwxact;
+	FdwXactOnDiskData	*fdwxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							fdw_part->usermapping->userid,
+							fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_INITIAL;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				Oid umid, char *fdwxact_id)
+{
+	int i;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->dbid = dbid;
+	fdwxact->local_xid = xid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	fdwxact->indoubt = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (FdwXactIsBeingResolved(fdwxact))
+		elog(ERROR, "cannot remove fdwxact entry that is beging resolved");
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true and set FdwXactAtomicCommitReady to true if the current transaction
+ * modified data on two or more servers in FdwXactParticipants and
+ * local server itself.
+ */
+static bool
+is_foreign_twophase_commit_required(void)
+{
+	ListCell*	lc;
+	int			nserverswritten = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	/*
+	 * Atomic commit is required if we modified data on two or more
+	 * participants.
+	 */
+	if (nserverswritten <= 1)
+		return false;
+
+	ForeignTwophaseCommitIsRequired = true;
+	return true;
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_lefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		if (!fdw_part->fdwxact)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in FdwXactParticipants
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget. So we need to check if the entries are still
+		 * associated with the transaction.
+		 */
+		SpinLockAcquire(&fdwxact->mutex);
+		if (fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+			fdwxact->indoubt = true;
+			n_lefts++;
+		}
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (n_lefts > 0)
+	{
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", n_lefts);
+		FdwXactComputeRequiredXmin();
+	}
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * Wait for the foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsForeignTwophaseCommitRequested())
+		return;
+
+	/*
+	 * Also, exit if the transaction itself has no foreign transaction
+	 * participants.
+	 */
+	if (FdwXactParticipants == NIL && wait_xid == MyPgXact->xid)
+		return;
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		foreach (lcell, FdwXactParticipants)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * If the foreign transaction has FdwXact entry we might have
+			 * prepared it. Skip already-prepared foreign transaction because
+			 * it has closed its transaction. But we are not sure that foreign
+			 * transaction with status == FDWXACT_STATUS_PREPARING has been
+			 * prepared or not. So we call the rollback API to close its
+			 * transaction for safety. The prepared foreign transaction that
+			 * we might have will be resolved by the foreign transaction
+			 * resolver.
+			 */
+			if (fdw_part->fdwxact)
+			{
+				bool is_prepared;
+
+				LWLockAcquire(FdwXactLock, LW_SHARED);
+				is_prepared = fdw_part->fdwxact &&
+					fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED;
+				LWLockRelease(FdwXactLock);
+
+				if (is_prepared)
+					continue;
+			}
+
+			/* One-phase rollback foreign transaction */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, false);
+		}
+	}
+
+	/*
+	 * In commit cases, we have already prepared foreign transactions during
+	 * pre-commit phase. And these prepared transactions will be resolved by
+	 * the resolver process.
+	 */
+
+	ForgetAllFdwXactParticipants();
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Prepare foreign transactions.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (is_foreign_twophase_commit_required() &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p)
+{
+	PGPROC *proc;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+			break;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+	{
+		*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+		*waitXid_p = proc->fdwXactWaitXid;
+	}
+	else
+	{
+		*nextResolutionTs_p = -1;
+		*waitXid_p = InvalidTransactionId;
+	}
+
+	LWLockRelease(FdwXactResolutionLock);
+
+	return proc;
+}
+
+/*
+ * Get one FdwXact entry to resolve. This function intended to be used when
+ * a resolver process get FdwXact entries to resolve. So we search entries
+ * while not including in-doubt transactions and in-progress transactions.
+ */
+static FdwXact
+get_fdwxact_to_resolve(Oid dbid, TransactionId xid)
+{
+	List *fdwxacts = NIL;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Don't include both in-doubt transactions and in-progress transactions */
+	fdwxacts = get_fdwxacts(dbid, xid, InvalidOid, InvalidOid,
+							false, false, false);
+
+	return fdwxacts == NIL ? NULL : (FdwXact) linitial(fdwxacts);
+}
+
+/*
+ * Resolve one distributed transaction on the given database . The target
+ * distributed transaction is fetched from the waiting queue and its transaction
+ * participants are fetched from the global array.
+ *
+ * Release the waiter and return true after we resolved the all of the foreign
+ * transaction participants. On failure, we re-enqueue the waiting backend after
+ * incremented the next resolution time.
+ */
+void
+FdwXactResolveTransactionAndReleaseWaiter(Oid dbid, TransactionId xid,
+										  PGPROC *waiter)
+{
+	FdwXact	fdwxact;
+
+	Assert(TransactionIdIsValid(xid));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	while ((fdwxact = get_fdwxact_to_resolve(MyDatabaseId, xid)) != NULL)
+	{
+		FdwXactRslvState *state;
+		ForeignServer *server;
+		UserMapping	*usermapping;
+
+		CHECK_FOR_INTERRUPTS();
+
+		server = GetForeignServer(fdwxact->serverid);
+		usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+
+		state = create_fdwxact_state();
+		SpinLockAcquire(&fdwxact->mutex);
+		state->server = server;
+		state->usermapping = usermapping;
+		state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+		SpinLockRelease(&fdwxact->mutex);
+
+		FdwXactDetermineTransactionFate(fdwxact, false);
+
+		/* Do not hold during foreign transaction resolution */
+		LWLockRelease(FdwXactLock);
+
+		PG_TRY();
+		{
+			/*
+			 * Resolve the foreign transaction. When committing or aborting
+			 * prepared foreign transactions the previous status is always
+			 * FDWXACT_STATUS_PREPARED.
+			 */
+			FdwXactResolveForeignTransaction(fdwxact, state,
+											 FDWXACT_STATUS_PREPARED);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDWXACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				waiter->fdwXactNextResolutionTs =
+					TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+												foreign_xact_resolution_retry_interval);
+				FdwXactQueueInsert(waiter);
+			}
+			LWLockRelease(FdwXactResolutionLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved one foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter
+	 * could already be detached if user cancelled to wait before
+	 * resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId	wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/* Wake up the waiter only when we have set state and removed from queue */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Determine whether the given foreign transaction should be committed or
+ * rolled back according to the result of the local transaction. This function
+ * changes fdwxact->status so the caller must hold FdwXactLock in exclusive
+ * mode or passing need_lock with true.
+ */
+static void
+FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock)
+{
+	bool			is_commit = false;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * The being resolved transaction must be either that has been cancelled
+	 *  and marked as in-doubt or that has been prepared.
+	 */
+	Assert(fdwxact->indoubt ||
+		   fdwxact->status == FDWXACT_STATUS_PREPARED);
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	if (TransactionIdDidCommit(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_COMMITTING;
+		is_commit = true;
+	}
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+		is_commit = false;
+	}
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+		is_commit = false;
+	}
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is
+	 * about to be committed or aborted. This should not happen except for one
+	 * case where the local transaction is prepared and this foreign transaction
+	 * is being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid),
+				 errhint("The local transaction with xid %u might be prepared",
+						 fdwxact->local_xid)));
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * callback function. The 'state' is passed to the callback function. The fate of
+ * foreign transaction must be determined. If foreign transaction is resolved
+ * successfully, remove the FdwXact entry from the shared memory and also
+ * remove the corresponding on-disk file. If failed, the status of FdwXact
+ * entry changes to 'fallback_status' before erroring out.
+ */
+static void
+FdwXactResolveForeignTransaction(FdwXact fdwxact, FdwXactRslvState *state,
+								 FdwXactStatus fallback_status)
+{
+	ForeignServer		*server;
+	ForeignDataWrapper	*fdw;
+	FdwRoutine			*fdw_routine;
+	bool				is_commit;
+
+	Assert(state != NULL);
+	Assert(state->server && state->usermapping && state->fdwxact_id);
+	Assert(fdwxact != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+		elog(ERROR, "cannot resolve foreign transaction whose fate is not determined");
+
+	is_commit = fdwxact->status == FDWXACT_STATUS_COMMITTING;
+	LWLockRelease(FdwXactLock);
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	PG_TRY();
+	{
+		if (is_commit)
+			fdw_routine->CommitForeignTransaction(state);
+		else
+			fdw_routine->RollbackForeignTransaction(state);
+	}
+	PG_CATCH();
+	{
+		/* Back to the fallback status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = fallback_status;
+		LWLockRelease(FdwXactLock);
+
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	elog(DEBUG1, "successfully %s the foreign transaction with xid %u db %u server %u user %u",
+		 is_commit ? "committed" : "rolled back",
+		 fdwxact->local_xid, fdwxact->dbid, fdwxact->serverid,
+		 fdwxact->userid);
+
+	fdwxact->status = FDWXACT_STATUS_RESOLVED;
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Return palloc'd and initialized FdwXactRslvState.
+ */
+static FdwXactRslvState *
+create_fdwxact_state(void)
+{
+	FdwXactRslvState *state;
+
+	state = palloc(sizeof(FdwXactRslvState));
+	state->server = NULL;
+	state->usermapping = NULL;
+	state->fdwxact_id = NULL;
+	state->flags = 0;
+
+	return state;
+}
+
+/*
+ * Return at least one FdwXact entry that matches to given argument,
+ * otherwise return NULL. All arguments must be valid values so that it can
+ * search exactly one (or none) entry. Note that this function intended to be
+ * used for modifying the returned FdwXact entry, so the caller must hold
+ * FdwXactLock in exclusive mode and it doesn't include the in-progress
+ * FdwXact entries.
+ */
+static FdwXact
+get_one_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	/* Include in-dbout transactions but don't include in-progress ones */
+	fdwxact_list = get_fdwxacts(dbid, xid, serverid, userid,
+								true, false, false);
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdwxact_list) <= 1);
+
+	/* Could not find entry */
+	if (fdwxact_list == NIL)
+		return NULL;
+
+	return (FdwXact) linitial(fdwxact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdwxact_exists(Oid dbid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	/* Find entries from all FdwXact entries */
+	fdwxact_list = get_fdwxacts(dbid, InvalidTransactionId, serverid,
+								userid, true, true, true);
+
+	return fdwxact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_foreign_xacts, and the number of entries to num_p.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't
+ * want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdwxacts(int *num_p)
+{
+	List		*all_fdwxacts;
+	ListCell	*lc;
+	FdwXact		fdwxacts;
+	int			num_fdwxacts = 0;
+
+	Assert(num_p != NULL);
+
+	/* Get all entries */
+	all_fdwxacts = get_fdwxacts(InvalidOid, InvalidTransactionId,
+								InvalidOid, InvalidOid, true,
+								true, true);
+
+	if (all_fdwxacts == NIL)
+	{
+		*num_p = 0;
+		return NULL;
+	}
+
+	fdwxacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdwxacts));
+	*num_p = list_length(all_fdwxacts);
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdwxacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdwxacts + num_fdwxacts, fx,
+			   sizeof(FdwXactData));
+		num_fdwxacts++;
+	}
+
+	list_free(all_fdwxacts);
+
+	return fdwxacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respecitively.
+ */
+static List*
+get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			 bool include_indoubt, bool include_in_progress, bool need_lock)
+{
+	int i;
+	List	*fdwxact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact	fdwxact = FdwXactCtl->fdwxacts[i];
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* include in-doubt transaction? */
+		if (!include_indoubt && fdwxact->indoubt)
+			continue;
+
+		/* include in-progress transaction? */
+		if (!include_in_progress && FdwXactIsBeingResolved(fdwxact))
+			continue;
+
+		/* Append it if matched */
+		fdwxact_list = lappend(fdwxact_list, fdwxact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdwxact_list;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	*id;
+	int		id_len = 0;
+
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		/*
+		 * FDW doesn't provide the callback function, generate an unique
+		 * idenetifier.
+		 */
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+			 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifer \"%s\" is too long",
+						id),
+				 errdetail("foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;						/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdwxacts,
+							 serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid =
+		XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid  != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							  fdwxact_data->serverid, fdwxact_data->userid,
+							  fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know
+	 * the xact status right now. Resolver will set it later based on
+	 * the status of local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;	/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdwxact(dbid, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		return;
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(int *newval, void **extra, GucSource source)
+{
+	ForeignTwophaseCommitLevel newForeignTwophaseCommitLevel = *newval;
+
+	/* Parameter check */
+	if (newForeignTwophaseCommitLevel > FOREIGN_TWOPHASE_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	7
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdwxacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdwxacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(PG_PREPARED_FDWXACTS_COLS);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "indoubt",
+						   BOOLOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 7, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdwxacts = get_all_fdwxacts(&num_fdwxacts);
+		status->num_xacts = num_fdwxacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdwxact = &status->fdwxacts[status->cur_xact++];
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdwxact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdwxact->dbid);
+		values[1] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[2] = ObjectIdGetDatum(fdwxact->serverid);
+		values[3] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (fdwxact->status)
+		{
+			case FDWXACT_STATUS_INITIAL:
+				xact_status = "initial";
+				break;
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			case FDWXACT_STATUS_RESOLVED:
+				xact_status = "resolved";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		values[5] = BoolGetDatum(fdwxact->indoubt);
+		values[6] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	FdwXact			fdwxact;
+	FdwXactRslvState	*state;
+	FdwXactStatus		prev_status;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	server = GetForeignServer(serverid);
+	usermapping = GetUserMapping(userid, serverid);
+	state = create_fdwxact_state();
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+	{
+		LWLockRelease(FdwXactLock);
+		PG_RETURN_BOOL(false);
+	}
+
+	state->server = server;
+	state->usermapping = usermapping;
+	state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+
+	SpinLockAcquire(&fdwxact->mutex);
+	prev_status = fdwxact->status;
+	SpinLockRelease(&fdwxact->mutex);
+
+	FdwXactDetermineTransactionFate(fdwxact, false);
+
+	LWLockRelease(FdwXactLock);
+
+	FdwXactResolveForeignTransaction(fdwxact, state, prev_status);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000..45fb530
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,644 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to retry resolution.
+ */
+void
+FdwXactLauncherRequestToLaunchForRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			resolver->last_resolved_time = 0;
+			resolver->latch = NULL;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a foreign_xact_resolution_retry_interval
+		 * but always starts when the backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find FdwXact
+		 * entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int unused_slot;
+	int i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (unused_slot > max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	*resolver_dbs;	/* DBs resolver's running on */
+	HTAB	*fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL	ctl;
+	HASH_SEQ_STATUS status;
+	Oid		*entry;
+	bool	launched;
+	int		i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->indoubt)
+			continue;
+
+		hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+		return false;
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign trasaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		 /* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+						WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+						10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Returns activity of all foreign transaction resolvers.
+ */
+Datum
+pg_stat_get_foreign_xact(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000..9298877
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,344 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+bool foreign_xact_resolve_indoubt_xacts;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	MyFdwXactResolver->last_resolved_time = 0;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		PGPROC			*waiter = NULL;
+		TransactionId	waitXid = InvalidTransactionId;
+		TimestampTz		resolutionTs = -1;
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or got the waiter
+		 * that has future resolution time.
+		 */
+		while ((waiter = FdwXactGetWaiter(&resolutionTs, &waitXid)) != NULL)
+		{
+			CHECK_FOR_INTERRUPTS();
+			Assert(TransactionIdIsValid(waitXid));
+
+			if	(resolutionTs > now)
+				break;
+
+			elog(DEBUG2, "resolver got one waiter with xid %u", waitXid);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveTransactionAndReleaseWaiter(MyDatabaseId, waitXid,
+													  waiter);
+			CommitTransactionCommand();
+
+			/* Update my stats */
+			SpinLockAcquire(&(MyFdwXactResolver->mutex));
+			MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+			SpinLockRelease(&(MyFdwXactResolver->mutex));
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz last_resolved_time;
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	last_resolved_time = MyFdwXactResolver->last_resolved_time;
+	timeout = TimestampTzPlusMilliseconds(last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * checked FdwXactWaiterExists.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long	sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index 5514db1..742e825 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -8,9 +8,9 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
-	   gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
-	   mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
-	   smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdwxactdesc.o \
+	genericdesc.o  gindesc.o gistdesc.o hashdesc.o heapdesc.o \
+	logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \
+	seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 33060f3..1d4e1c8 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47..f15c83a 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -12,9 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
-	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
-	xact.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o generic_xlog.o multixact.o \
+	parallel.o rmgr.o slru.o subtrans.o timeline.o transam.o twophase.o \
+	twophase_rmgr.o varsup.o xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9368b56..8b360b1 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -9,6 +9,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
 #include "access/generic_xlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 5196d61..dfc4d1d 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -852,6 +853,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 }
 
 /*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
+/*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
  *
@@ -2318,6 +2348,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2377,6 +2413,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index d7930c0..8aaf5ae 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1216,6 +1217,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1224,6 +1226,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1262,12 +1265,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1425,6 +1429,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2084,6 +2096,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2243,6 +2258,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2330,6 +2346,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2524,6 +2542,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2729,6 +2748,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 13e0d23..9740855 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
 #include "access/subtrans.h"
@@ -5249,6 +5250,7 @@ BootStrapXLOG(void)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6179,6 +6181,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -6701,14 +6706,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -6900,7 +6906,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7396,6 +7405,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7714,6 +7724,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -8989,6 +9002,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9422,8 +9436,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9441,6 +9457,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9457,6 +9474,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9662,6 +9680,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -9861,6 +9880,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index ea4c85e..342dd6a 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -332,6 +332,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
@@ -815,6 +818,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_foreign_xact AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_foreign_xact() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index f1161f0..ef0078c 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2857,9 +2857,16 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
 
+	}
+
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
 
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index d7bc6e3..76f7b78 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1105,6 +1107,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1424,6 +1438,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
 	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
+	/*
 	 * Do the deletion
 	 */
 	object.classId = UserMappingRelationId;
@@ -1576,6 +1599,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index 6f2b4d6..16fab44 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,7 +13,9 @@
  */
 #include "postgres.h"
 
+
 #include "access/table.h"
+#include "access/fdwxact.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
 #include "catalog/pg_inherits.h"
@@ -949,7 +951,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 52af1da..eb7450c 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,10 +226,33 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
 
+	}
+
 	return scanstate;
 }
 
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index a70549d..d942ea5 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -38,6 +38,7 @@
 #include "postgres.h"
 
 #include "access/heapam.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
 #include "access/xact.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -550,6 +552,10 @@ ExecInsert(ModifyTableState *mtstate,
 										   NULL,
 										   specToken);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -778,6 +784,10 @@ ldelete:;
 									&tmfd,
 									changingPart);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -1325,6 +1335,10 @@ lreplace:;
 									true /* wait for commit */ ,
 									&tmfd, &lockmode, &update_indexes);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -2387,6 +2401,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index c917ec4..2780ed5 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMapping - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,20 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction &&
+		 !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction &&
+		 routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data-wrapper must support both commit and rollback routine or either");
+
+	if (routine->PrepareForeignTransaction &&
+		(!routine->CommitForeignTransaction ||
+		 !routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index b66b517..517169b 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -15,6 +15,8 @@
 #include <unistd.h>
 
 #include "libpq/pqsignal.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index b4f2b28..44b8ebd 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3652,6 +3652,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3855,6 +3861,11 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDWXACT:
+			event_name = "FdwXact";
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -4070,6 +4081,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 688ad43..b840eff 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -900,6 +902,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -975,12 +981,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index 151c3ef..325cf9d 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -154,6 +154,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index d7d7335..1491bc6 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(int port)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(int port)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 18a0f62..59dfc5f 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -91,6 +91,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -246,6 +248,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1313,6 +1316,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1378,6 +1382,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1428,6 +1433,15 @@ GetOldestXmin(Relation rel, int flags)
 		result = replication_slot_xmin;
 
 	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
+	/*
 	 * After locks have been released and defer_cleanup_age has been applied,
 	 * check whether we need to back up further to make logical decoding
 	 * possible. We need to do so if we're computing the global limit (rel =
@@ -3016,6 +3030,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index db47843..0b8a487 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -49,3 +49,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 CLogTruncationLock					44
+FdwXactLock					45
+FdwXactResolverLock			46
+FdwXactResolutionLock			47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 498373f..dc77509 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -422,6 +423,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -823,6 +828,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 44a59e1..8b8330b 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3000,6 +3002,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 631f16f..ec3a45d 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -397,6 +398,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 };
 
 /*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
+/*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
  */
@@ -719,6 +739,12 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
+	/* FDWXACT_RESOLVER */
+	gettext_noop("Foreign Transaction Management / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2352,6 +2378,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FDWXACT_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FDWXACT_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4374,6 +4446,16 @@ static struct config_enum ConfigureNamesEnum[] =
 	},
 
 	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
+	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
 			NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 5ee5e09..da31b2b 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -341,6 +343,20 @@
 
 
 #------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
+#------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index 33ac627..328b857 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 2ef1791..3659501 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -210,6 +210,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index d955b97..20678e4 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -310,6 +310,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 2734f87..840df85 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -717,6 +717,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -933,6 +934,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca..b616cea 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 3c0db2c..5798b4c 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Foreign Transactions", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index b9a531c..8238723 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index c9d6b47..2e6da26 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -109,6 +109,13 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 3cc9c3d..1410a11 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -228,6 +228,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index ff98d9e..773846d 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 8733524..abed76f 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5150,6 +5150,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '6053', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_foreign_xact', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_foreign_xact' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5863,6 +5870,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '6050', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o,o}',
+  proargnames => '{dbid,xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '6051', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '6052', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -5981,6 +6006,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '6054',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 8226860..f6592ee 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
@@ -248,7 +260,6 @@ typedef struct FdwRoutine
 	ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild;
 } FdwRoutine;
 
-
 /* Functions in foreign/foreign.c */
 extern FdwRoutine *GetFdwRoutine(Oid fdwhandler);
 extern Oid	GetForeignServerIdByRelId(Oid relid);
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 46759e3..4a150e6 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -80,6 +80,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 											   bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 														 bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index 0a3ad3a..33d34d2 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -777,6 +777,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -854,7 +856,9 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDWXACT,
+	WAIT_EVENT_FDWXACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -934,6 +938,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index ac7ee72..04111f9 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -21,6 +21,7 @@
 #include "storage/lock.h"
 #include "storage/pg_sema.h"
 #include "storage/proclist_types.h"
+#include "datatype/timestamp.h"
 
 /*
  * Each backend advertises up to PGPROC_MAX_CACHED_SUBXIDS TransactionIds
@@ -153,6 +154,16 @@ struct PGPROC
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
+	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
 	 * their lock.
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index da8b672..04f9c8c 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -124,4 +126,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index d68976f..d5fec50 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,9 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
+	FDWXACT_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 210e9cd..c862e0e 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1341,6 +1341,14 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.dbid,
+    f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(dbid, xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
@@ -1841,6 +1849,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_foreign_xact| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_foreign_xact() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_gssapi| SELECT s.pid,
     s.gss_auth AS gss_authenticated,
     s.gss_princ AS principal,
-- 
2.10.5

From 45bbe2360c19e415e004a6aa512d968ee26649e3 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Tue, 2 Jul 2019 09:31:05 +0900
Subject: [PATCH v24 3/5] Documentation update.

---
 doc/src/sgml/catalogs.sgml                | 145 ++++++++++++++++++
 doc/src/sgml/config.sgml                  | 146 +++++++++++++++++-
 doc/src/sgml/distributed-transaction.sgml | 158 ++++++++++++++++++++
 doc/src/sgml/fdwhandler.sgml              | 236 ++++++++++++++++++++++++++++++
 doc/src/sgml/filelist.sgml                |   1 +
 doc/src/sgml/func.sgml                    |  89 +++++++++++
 doc/src/sgml/monitoring.sgml              |  60 ++++++++
 doc/src/sgml/postgres.sgml                |   1 +
 doc/src/sgml/storage.sgml                 |   6 +
 9 files changed, 841 insertions(+), 1 deletion(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 09690b6..4fe4b1c 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8274,6 +8274,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
      </row>
 
      <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
+     <row>
       <entry><link linkend="view-pg-file-settings"><structname>pg_file_settings</structname></link></entry>
       <entry>summary of configuration file contents</entry>
      </row>
@@ -9718,6 +9723,146 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with that this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on that the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>initial</literal> : Initial status.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>resolved</literal> : This foreign transaction has been resolved.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-dbout status and
+       needs to be resolved by calling <function>pg_resolve_fdwxact</function>
+       function.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 84341a3..3e988e3 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4281,7 +4281,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -8615,6 +8614,151 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign
+         transaction to be resolved before the command returns a "success"
+         indication to the client. Valid values are <literal>required</literal>,
+         <literal>prefer</literal> and <literal>disabled</literal>. The default
+         setting is <literal>disabled</literal>. Setting to
+         <literal>disabled</literal> don't use two-phase commit protocol to
+         commit or rollback distributed transactions. When set to
+         <literal>required</literal> the distributed transaction strictly
+         requires that all written servers can use two-phase commit protocol.
+         That is, the distributed transaction cannot commit if even one server
+         does not support the transaction management callback routines
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. Note that when <literal>disabled</literal> or
+         <literal>prefer</literal> there can be risk of database consistency
+         among all servers that involved in the distributed transaction when some
+         foreign server crashes during committing the distributed transaction.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero
+         value to set this parameter either <literal>required</literal> or
+         <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000..350b1af
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,158 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).
+    A <productname>PostgreSQL</productname> server that received SQL is called
+    <firstterm>coordinator node</firstterm> who is responsible for coordinating
+    all the partipanting transactions. Using two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed
+    transaction manager prepares all transaction on the foreign servers if
+    two-phase commit is required. Two-phase commit is required when the
+    transaction modifies data on two or more servers including the local server
+    itself and <xref linkend="guc-foreign-twophase-commit"/>is
+    <literal>required</literal> or <literal>prefer</literal>. If all preparations
+    on foreign servers got successful go to the next step. Any failure happens
+    in this step <productname>PostgreSQL</productname> changes to rollback, then
+    rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the
+    transaction locally. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes rollback, then rollback all
+    transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolution">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that is
+    responsible for foreign transaction resolution. They commit or rollback all
+    prepared transaction on foreign servers if the coordinator received agreement
+    messages from all foreign servers during the first step.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on one database of the coordinator side. On failure during resolution, they
+    retries to resolve at an interval of
+    <varname>foreign_transaction_resolution_interval</varname> time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped. So to drop the database, you can call
+     <function>pg_stop_foreign_xact_resovler</function> function before dropping
+     the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>Manual Resolution of In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, distributed transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the coordinator node crashed during either preparing or
+    resolving distributed transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="pg-stat-foreign-xact-view"/>
+    view. These foreign transactions need to be resolved by using
+    <function>pg_resolve_foriegn_xact</function> function.
+    <productname>PostgreSQL</productname> doesn't have facilities to automatially
+    resolve in-doubt transactions. These behavior might change in a future release.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-monitoring">
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-foreign-xact-view"><literal>pg_stat_foreign_xact</literal></link>
+    view. This view contains one row for every foreign transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 27b94fb..1b292e4 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1413,6 +1413,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1892,4 +2013,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 3da2365..80a87fa 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 3a8581d..f03c4cd 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -21996,6 +21996,95 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index bf72d0c..d587164 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -368,6 +368,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_foreign_xact</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1236,6 +1244,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
+        <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
         </row>
@@ -1459,6 +1479,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
+        <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
          <entry>Waiting during base backup when throttling activity.</entry>
@@ -2338,6 +2362,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-view" xreflabel="pg_stat_foreign_xact">
+   <title><structname>pg_stat_foreign_xact</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index 3e115f1..5ae3807 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -163,6 +163,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 1047c77..f69bc72 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -84,6 +84,12 @@ Item
 </row>
 
 <row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
+<row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
 </row>
-- 
2.10.5

From 9f7cbe1f78c550bf62f435ff97845f7fde5f46cd Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:46:01 +0900
Subject: [PATCH v24 4/5] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/Makefile                  |   7 +-
 contrib/postgres_fdw/connection.c              | 604 ++++++++++++++++---------
 contrib/postgres_fdw/expected/postgres_fdw.out | 261 ++++++++++-
 contrib/postgres_fdw/fdwxact.conf              |   3 +
 contrib/postgres_fdw/postgres_fdw.c            |  21 +-
 contrib/postgres_fdw/postgres_fdw.h            |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql      | 119 +++++
 doc/src/sgml/postgres-fdw.sgml                 |  46 ++
 8 files changed, 830 insertions(+), 238 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index 85394b4..5198f40 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -10,7 +10,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -23,3 +23,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index 57ed5f4..093eda6 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2019, PostgreSQL Global Development Group
  *
@@ -14,6 +14,7 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "catalog/pg_user_mapping.h"
@@ -56,6 +57,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -69,17 +71,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 								   SubTransactionId mySubid,
 								   SubTransactionId parentSubid,
@@ -91,24 +89,26 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 									 bool ignore_errors);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 									 PGresult **result);
-
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +128,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,12 +135,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -155,6 +148,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -182,6 +190,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +199,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +210,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,11 +226,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -414,7 +461,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -645,193 +692,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 }
 
 /*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow remote transactions that modified anything,
-					 * since it's not very reasonable to hold them open until
-					 * the prepared transaction is committed.  For the moment,
-					 * throw error unconditionally; later we might allow
-					 * read-only cases.  Note that the error will cause us to
-					 * come right back here with event == XACT_EVENT_ABORT, so
-					 * we'll clean up the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot prepare a transaction that modified remote tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
-/*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
 static void
@@ -847,10 +707,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -861,6 +717,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1195,3 +1055,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recusively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index f0c842a..067c98f 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -179,15 +198,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8781,3 +8802,225 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
+ count 
+-------
+     0
+(1 row)
+
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000..3fdbf93
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 1759b9e..b672786 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -503,7 +504,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 							  const PgFdwRelationInfo *fpinfo_o,
 							  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -557,6 +557,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1446,7 +1451,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2384,7 +2389,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2684,7 +2689,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3512,7 +3517,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4387,7 +4392,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4477,7 +4482,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4700,7 +4705,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index 6acb7dc..f9162bb 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/pathnodes.h"
@@ -127,7 +128,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -135,6 +136,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 							   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int	ExtractConnectionOptions(List *defelems,
@@ -201,6 +205,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 									bool is_subquery,
 									List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 630b803..572077c 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2479,3 +2502,99 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index e9ce39a..0427922 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -441,6 +441,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -469,6 +506,14 @@
   </para>
 
   <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
+  <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
    isolation level; otherwise it uses <literal>REPEATABLE READ</literal>
@@ -483,6 +528,7 @@
    COMMITTED</literal> local transaction.  A future
    <productname>PostgreSQL</productname> release might modify these rules.
   </para>
+
  </sect2>
 
  <sect2>
-- 
2.10.5

From e6450e9f000dfe922bd71e3c4adc9b16e1c45a6e Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:48:08 +0900
Subject: [PATCH v24 5/5] Add regression tests for atomic commit.

---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index e66e695..b17429f 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000..9af9bb8
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 117a954..dc8150a 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2319,9 +2319,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2336,7 +2339,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.10.5

From 13d2a332b84c38a1ccdd3705977e3a0859028bf5 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Fri, 8 Feb 2019 10:44:54 +0900
Subject: [PATCH v25 1/5] Keep track of writing on non-temporary relation

---
 src/backend/executor/nodeModifyTable.c | 12 ++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 18 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 01fe11a..778ff27 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -588,6 +588,10 @@ ExecInsert(ModifyTableState *mtstate,
 							   estate->es_output_cid,
 							   0, NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -940,6 +944,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1451,6 +1459,10 @@ lreplace:;
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL, NIL);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	if (canSetTag)
 		(estate->es_processed)++;
 
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index d714551..6f4013e 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -103,6 +103,12 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
 /*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
-- 
2.10.5

From 9a9893d9bdb65fdc76405100dd712174b00e4e5c Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:48:08 +0900
Subject: [PATCH v25 5/5] Add regression tests for atomic commit.

---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index e66e695..b17429f 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000..9af9bb8
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index b4045ab..022ba1b 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2323,9 +2323,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2340,7 +2343,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.10.5

From eba9ba70c673a6e0716c6b1566255bd3097add32 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:46:01 +0900
Subject: [PATCH v25 4/5] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/Makefile                  |   7 +-
 contrib/postgres_fdw/connection.c              | 604 ++++++++++++++++---------
 contrib/postgres_fdw/expected/postgres_fdw.out | 261 ++++++++++-
 contrib/postgres_fdw/fdwxact.conf              |   3 +
 contrib/postgres_fdw/postgres_fdw.c            |  21 +-
 contrib/postgres_fdw/postgres_fdw.h            |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql      | 119 +++++
 doc/src/sgml/postgres-fdw.sgml                 |  46 ++
 8 files changed, 830 insertions(+), 238 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index 85394b4..5198f40 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -10,7 +10,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -23,3 +23,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index 57ed5f4..093eda6 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2019, PostgreSQL Global Development Group
  *
@@ -14,6 +14,7 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "catalog/pg_user_mapping.h"
@@ -56,6 +57,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -69,17 +71,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 								   SubTransactionId mySubid,
 								   SubTransactionId parentSubid,
@@ -91,24 +89,26 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 									 bool ignore_errors);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 									 PGresult **result);
-
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +128,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,12 +135,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -155,6 +148,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -182,6 +190,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +199,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +210,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,11 +226,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -414,7 +461,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -645,193 +692,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 }
 
 /*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow remote transactions that modified anything,
-					 * since it's not very reasonable to hold them open until
-					 * the prepared transaction is committed.  For the moment,
-					 * throw error unconditionally; later we might allow
-					 * read-only cases.  Note that the error will cause us to
-					 * come right back here with event == XACT_EVENT_ABORT, so
-					 * we'll clean up the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot prepare a transaction that modified remote tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
-/*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
 static void
@@ -847,10 +707,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -861,6 +717,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1195,3 +1055,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recusively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index f0c842a..067c98f 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -179,15 +198,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8781,3 +8802,225 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
+ count 
+-------
+     0
+(1 row)
+
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000..3fdbf93
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 82d8140..80de315 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -503,7 +504,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 							  const PgFdwRelationInfo *fpinfo_o,
 							  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -557,6 +557,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1446,7 +1451,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2384,7 +2389,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2684,7 +2689,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3512,7 +3517,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4387,7 +4392,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4477,7 +4482,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4705,7 +4710,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index 6acb7dc..f9162bb 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/pathnodes.h"
@@ -127,7 +128,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -135,6 +136,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 							   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int	ExtractConnectionOptions(List *defelems,
@@ -201,6 +205,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 									bool is_subquery,
 									List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 630b803..572077c 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2479,3 +2502,99 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index e9ce39a..0427922 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -441,6 +441,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -469,6 +506,14 @@
   </para>
 
   <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
+  <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
    isolation level; otherwise it uses <literal>REPEATABLE READ</literal>
@@ -483,6 +528,7 @@
    COMMITTED</literal> local transaction.  A future
    <productname>PostgreSQL</productname> release might modify these rules.
   </para>
+
  </sect2>
 
  <sect2>
-- 
2.10.5

From dcbe971ca77cd536d4a4e78492ad03d36f0edb6b Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Tue, 2 Jul 2019 09:31:05 +0900
Subject: [PATCH v25 3/5] Documentation update.

---
 doc/src/sgml/catalogs.sgml                | 145 ++++++++++++++++++
 doc/src/sgml/config.sgml                  | 146 +++++++++++++++++-
 doc/src/sgml/distributed-transaction.sgml | 158 ++++++++++++++++++++
 doc/src/sgml/fdwhandler.sgml              | 236 ++++++++++++++++++++++++++++++
 doc/src/sgml/filelist.sgml                |   1 +
 doc/src/sgml/func.sgml                    |  89 +++++++++++
 doc/src/sgml/monitoring.sgml              |  60 ++++++++
 doc/src/sgml/postgres.sgml                |   1 +
 doc/src/sgml/storage.sgml                 |   6 +
 9 files changed, 841 insertions(+), 1 deletion(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 5e71a2e..1dbc45e 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8274,6 +8274,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
      </row>
 
      <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
+     <row>
       <entry><link linkend="view-pg-file-settings"><structname>pg_file_settings</structname></link></entry>
       <entry>summary of configuration file contents</entry>
      </row>
@@ -9718,6 +9723,146 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with that this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on that the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>initial</literal> : Initial status.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>resolved</literal> : This foreign transaction has been resolved.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-dbout status and
+       needs to be resolved by calling <function>pg_resolve_fdwxact</function>
+       function.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 89284dc..c1ef5d8 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4294,7 +4294,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -8611,6 +8610,151 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign
+         transaction to be resolved before the command returns a "success"
+         indication to the client. Valid values are <literal>required</literal>,
+         <literal>prefer</literal> and <literal>disabled</literal>. The default
+         setting is <literal>disabled</literal>. Setting to
+         <literal>disabled</literal> don't use two-phase commit protocol to
+         commit or rollback distributed transactions. When set to
+         <literal>required</literal> the distributed transaction strictly
+         requires that all written servers can use two-phase commit protocol.
+         That is, the distributed transaction cannot commit if even one server
+         does not support the transaction management callback routines
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. Note that when <literal>disabled</literal> or
+         <literal>prefer</literal> there can be risk of database consistency
+         among all servers that involved in the distributed transaction when some
+         foreign server crashes during committing the distributed transaction.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero
+         value to set this parameter either <literal>required</literal> or
+         <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000..350b1af
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,158 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).
+    A <productname>PostgreSQL</productname> server that received SQL is called
+    <firstterm>coordinator node</firstterm> who is responsible for coordinating
+    all the partipanting transactions. Using two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed
+    transaction manager prepares all transaction on the foreign servers if
+    two-phase commit is required. Two-phase commit is required when the
+    transaction modifies data on two or more servers including the local server
+    itself and <xref linkend="guc-foreign-twophase-commit"/>is
+    <literal>required</literal> or <literal>prefer</literal>. If all preparations
+    on foreign servers got successful go to the next step. Any failure happens
+    in this step <productname>PostgreSQL</productname> changes to rollback, then
+    rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the
+    transaction locally. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes rollback, then rollback all
+    transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolution">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that is
+    responsible for foreign transaction resolution. They commit or rollback all
+    prepared transaction on foreign servers if the coordinator received agreement
+    messages from all foreign servers during the first step.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on one database of the coordinator side. On failure during resolution, they
+    retries to resolve at an interval of
+    <varname>foreign_transaction_resolution_interval</varname> time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped. So to drop the database, you can call
+     <function>pg_stop_foreign_xact_resovler</function> function before dropping
+     the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>Manual Resolution of In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, distributed transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the coordinator node crashed during either preparing or
+    resolving distributed transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="pg-stat-foreign-xact-view"/>
+    view. These foreign transactions need to be resolved by using
+    <function>pg_resolve_foriegn_xact</function> function.
+    <productname>PostgreSQL</productname> doesn't have facilities to automatially
+    resolve in-doubt transactions. These behavior might change in a future release.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-monitoring">
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-foreign-xact-view"><literal>pg_stat_foreign_xact</literal></link>
+    view. This view contains one row for every foreign transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 27b94fb..1b292e4 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1413,6 +1413,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1892,4 +2013,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 3da2365..80a87fa 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index c878a0b..4d09fa0 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -22013,6 +22013,95 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index bf72d0c..d587164 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -368,6 +368,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_foreign_xact</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1236,6 +1244,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
+        <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
         </row>
@@ -1459,6 +1479,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
+        <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
          <entry>Waiting during base backup when throttling activity.</entry>
@@ -2338,6 +2362,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-view" xreflabel="pg_stat_foreign_xact">
+   <title><structname>pg_stat_foreign_xact</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index e59cba7..dee3f72 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -163,6 +163,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 342a0ff..840b0f6 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -84,6 +84,12 @@ Item
 </row>
 
 <row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
+<row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
 </row>
-- 
2.10.5

From 495152c6204554384f3cc92da4880377cd507213 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Tue, 2 Jul 2019 09:32:16 +0900
Subject: [PATCH v25 2/5] Support atomic commit among multiple foreign servers.

---
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  130 ++
 src/backend/access/fdwxact/fdwxact.c          | 2827 +++++++++++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  644 ++++++
 src/backend/access/fdwxact/resolver.c         |  344 +++
 src/backend/access/rmgrdesc/Makefile          |    8 +-
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/Makefile           |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/copy.c                   |    7 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    9 +
 src/backend/executor/nodeForeignscan.c        |   25 +
 src/backend/executor/nodeModifyTable.c        |   18 +
 src/backend/foreign/foreign.c                 |   57 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   20 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   82 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   29 +
 src/include/foreign/fdwapi.h                  |   13 +-
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    9 +-
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    3 +
 src/test/regress/expected/rules.out           |   13 +
 49 files changed, 4540 insertions(+), 26 deletions(-)
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100755 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c

diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a..49480dd 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000..0207a66
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000..a6a46ad
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,130 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a conssitent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+--------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consisnts
+of the following four steps:
+
+1. Foriegn Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+we record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE all foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared tranascation on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step(commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itselft to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rolbacked the resolver process wake up the waiter.
+
+
+API Contract With Transaction Management Callback Functions
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForiegnTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for either PREPARE, COMMIT or
+ROLLBACK the trasaction on the foreign server respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transactio in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions has an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_INITIAL
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_PREPARING, FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING
+before the foreign transaction is prepared, committed and aborted by FDW
+callback functions respectively(*1). And the status then changes to
+FDWXACT_STATUS_RESOLVED once the foreign transaction are resolved, and then
+the corresponding FdwXact entry is removed with WAL logging. If failed during
+processing foreign transaction (i.g. preparing, committing or aborting) the
+status changes back to the previous status. Therefore the status
+FDWXACT_STATUS_xxxING appear only during the foreign transaction is being
+processed by an FDW callback function.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*2). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                      INVALID                       |
+ +----------------------------------------------------+
+    |                      |                       |
+    |                      v                       |
+    |           +---------------------+            |
+    |           |       INITIAL       |            |
+    |           +---------------------+            |
+   (*2)                    |                      (*2)
+    |                      v                       |
+    |           +---------------------+            |
+    |           |    PREPARING(*1)    |            |
+    |           +---------------------+            |
+    |                      |                       |
+    v                      v                       v
+ +----------------------------------------------------+
+ |                      PREPARED                      |
+ +----------------------------------------------------+
+           |                               |
+           v                               v
+ +--------------------+          +--------------------+
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |
+ +--------------------+          +--------------------+
+           |                               |
+           v                               v
+ +----------------------------------------------------+
+ |                      RESOLVED                      |
+ +----------------------------------------------------+
+
+(*1) Status that appear only during being processed by FDW
+(*2) Paths for recovered FdwXact entries
\ No newline at end of file
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100755
index 0000000..e3754e0
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2827 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers automically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit. The foreign servers are
+ * registered if FDW has both CommitForeignTransaciton API and
+ * RollbackForeignTransactionAPI. Registered participant servers are identified
+ * by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request. We have one queue
+ * of which elements are ordered by the timestamp that they expect to be
+ * processed at. Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp that they expects to be processed.
+ * Similary if failed to resolve them, it enqueues again with new timestamp
+ * (its timestamp + foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction). Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by pg_resovle_fdwxact()
+ * function.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or
+ * more servers including itself. In other case, all foreign transactions are
+ * committed or rolled back during pre-commit.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed by FDW, the corresponding
+ * FdwXact entry is update. In order to protect the entry from concurrent
+ * removing we need to hold a lock on the entry or a lock for entire global
+ * array. However, we don't want to hold the lock during FDW is processing the
+ * foreign transaction that may take a unpredictable time. To avoid this, the
+ * in-memory data of foreign transaction follows a locking model based on
+ * four linked concepts:
+ *
+ * * A foreign transaction's status variable is switched using the LWLock
+ *   FdwXactLock, which need to be hold in exclusive mode when updating the
+ *   status, while readers need to hold it in shared mode when looking at the
+ *   status.
+ * * A process who is going to update FdwXact entry cannot process foreign
+ *   transaction that is being resolved.
+ * * So setting the status to FDWACT_STATUS_PREPARING,
+ *   FDWXACT_STATUS_COMMITTING or FDWXACT_STATUS_ABORTING, which makes foreign
+ *   transaction in-progress states, means to own the FdwXact entry, which
+ *   protect it from updating/removing by concurrent writers.
+ * * Individual fields are protected by mutex where only the backend owning
+ *   the foreign transaction is authorized to update the fields from its own
+ *   one.
+
+ * Therefore, before doing PREPARE, COMMIT PREPARED or ROLLBACK PREPARED a
+ * process who is going to call transaction callback functions needs to change
+ * the status to the corresponding status above while holding FdwXactLock in
+ * exclusive mode, and call callback function after releasing the lock.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk. FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon. We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts. If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to dink.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Atomic commit is enabled by configuration */
+#define IsForeignTwophaseCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	(IsForeignTwophaseCommitEnabled() && \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED))
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define IsSeverCapableOfTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/* Check the FdwXact is begin resolved */
+#define FdwXactIsBeingResolved(fx) \
+	(((((FdwXact)(fx))->status) == FDWXACT_STATUS_PREPARING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_COMMITTING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_ABORTING))
+
+/*
+ * Structure to bundle the foreign transaction participant. This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry
+	 * is not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char			*fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool			modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact_fn;
+	CommitForeignTransaction_function	commit_foreign_xact_fn;
+	RollbackForeignTransaction_function	rollback_foreign_xact_fn;
+	GetPrepareId_function				get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit. This list
+ * has only foreign servers that provides transaction management callbacks,
+ * that is CommitForeignTransaction and RollbackForeignTransaction.
+ */
+static List *FdwXactParticipants = NIL;
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+int foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid,
+										 FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+												 bool for_commit);
+static void FdwXactResolveForeignTransaction(FdwXact fdwxact,
+											 FdwXactRslvState *state,
+											 FdwXactStatus fallback_status);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid,	void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static void FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock);
+static bool is_foreign_twophase_commit_required(void);
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static List *get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						  bool including_indoubts, bool include_in_progress,
+						  bool need_lock);
+static FdwXact get_all_fdwxacts(int *num_p);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXact get_fdwxact_to_resolve(Oid dbid, TransactionId xid);
+static FdwXactRslvState *create_fdwxact_state(void);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation		rel;
+	Oid				serverid;
+	Oid				userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign transaction identified by given server id and user id.
+ * Registered foreign transactions are managed by the global transaction
+ * manager until the end of the transaction.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant	*fdw_part;
+	ForeignServer 		*foreign_server;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	MemoryContext		old_ctx;
+	FdwRoutine			*routine;
+	ListCell	   		*lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Don't register foreign server if it doesn't provide both commit and
+	 * rollback transaction management callbacks.
+	 */
+	if (!routine->CommitForeignTransaction ||
+		!routine->RollbackForeignTransaction)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		pfree(routine);
+		return;
+	}
+
+	/*
+	 * Remember we touched the foreign server that is not capable of two-phase
+	 * commit.
+	 */
+	if (!routine->PrepareForeignTransaction)
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+	foreign_server = GetForeignServer(serverid);
+	fdw = GetForeignDataWrapper(foreign_server->fdwid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&(fdwxacts[cnt].mutex));
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * If foreign twophase commit is required, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign server's FDWs to support two-phase commit protocol and ask them to
+ *  prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and for 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_twophase_commit;
+	ListCell	*lc = NULL;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * we require all modified server have to be capable of two-phase
+	 * commit protocol.
+	 */
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false
+	 * if foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = is_foreign_twophase_commit_required();
+
+	/*
+	 * Firstly, we consider to commit foreign transactions in one-phase.
+	 */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool	commit = false;
+
+		/* Can commit in one-phase if two-phase commit is not requried */
+		if (!need_twophase_commit)
+			commit = true;
+
+		/* Non-modified foreign transaction always can be committed in one-phase */
+		if (!fdw_part->modified)
+			commit = true;
+
+		/*
+		 * In 'prefer' case, non-twophase-commit capable server can be
+		 * committed in one-phase.
+		 */
+		if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER &&
+			!IsSeverCapableOfTwophaseCommit(fdw_part))
+			commit = true;
+
+		if (commit)
+		{
+			/* Commit the foreign transaction in one-phase */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, true);
+
+			/* Delete it from the participant list */
+			FdwXactParticipants = foreach_delete_current(FdwXactParticipants,
+														 lc);
+			continue;
+		}
+	}
+
+	/* All done if we committed all foreign transactions */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * Secondary, if only one transaction is remained in the participant list
+	 * and we didn't modified the local data we can commit it without
+	 * preparation.
+	 */
+	if (list_length(FdwXactParticipants) == 1 &&
+		(MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) == 0)
+	{
+		/* Commit the foreign transaction in one-phase */
+		FdwXactOnePhaseEndForeignTransaction(linitial(FdwXactParticipants),
+											 true);
+
+		/* All foreign transaction must be committed */
+		list_free(FdwXactParticipants);
+		return;
+	}
+
+	/*
+	 * Finally, prepare foreign transactions. Note that we keep
+	 * FdwXactParticipants until the end of transaction.
+	 */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here. If any error occurs, we rollback
+ * non-prepared foreign trasactions and leave others to the resolver.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell		*lcell;
+	TransactionId	xid;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	xid = GetTopTransactionId();
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXactRslvState 	*state;
+		FdwXact		fdwxact;
+
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, xid);
+
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the FDWXACT_STATUS_PREPARING
+		 * status. Registration persists this information to the disk and logs
+		 * (that way relaying it on standby). Thus in case we loose connectivity
+		 * to the foreign server or crash ourselves, we will remember that we
+		 * might have prepared transaction on the foreign server and try to
+		 * resolve it when connectivity is restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(xid, fdw_part);
+
+		state = create_fdwxact_state();
+		state->server = fdw_part->server;
+		state->usermapping = fdw_part->usermapping;
+		state->fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+
+		/* Update the status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		Assert(fdwxact->status == FDWXACT_STATUS_INITIAL);
+		fdwxact->status = FDWXACT_STATUS_PREPARING;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 *
+		 * During abort processing, we might try to resolve a never-preapred
+		 * transaction, and get an error. This is fine as long as the FDW
+		 * provides us unique prepared transaction identifiers.
+		 */
+		PG_TRY();
+		{
+			fdw_part->prepare_foreign_xact_fn(state);
+		}
+		PG_CATCH();
+		{
+			/* failed, back to the initial state */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			fdwxact->status = FDWXACT_STATUS_INITIAL;
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		/* succeeded, update status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * One-phase commit or rollback the given foreign transaction participant.
+ */
+static void
+FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+									 bool for_commit)
+{
+	FdwXactRslvState *state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state = create_fdwxact_state();
+	state->server = fdw_part->server;
+	state->usermapping = fdw_part->usermapping;
+	state->flags = FDWXACT_FLAG_ONEPHASE;
+
+	/*
+	 * Commit or rollback foreign transaction in one-phase. Since we didn't
+	 * insert FdwXact entry for this transaction we don't need to care
+	 * failures. On failure we change to rollback.
+	 */
+	if (for_commit)
+		fdw_part->commit_foreign_xact_fn(state);
+	else
+		fdw_part->rollback_foreign_xact_fn(state);
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fdwxact;
+	FdwXactOnDiskData	*fdwxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							fdw_part->usermapping->userid,
+							fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_INITIAL;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				Oid umid, char *fdwxact_id)
+{
+	int i;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->dbid = dbid;
+	fdwxact->local_xid = xid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	fdwxact->indoubt = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (FdwXactIsBeingResolved(fdwxact))
+		elog(ERROR, "cannot remove fdwxact entry that is beging resolved");
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true and set FdwXactAtomicCommitReady to true if the current transaction
+ * modified data on two or more servers in FdwXactParticipants and
+ * local server itself.
+ */
+static bool
+is_foreign_twophase_commit_required(void)
+{
+	ListCell*	lc;
+	int			nserverswritten = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	/*
+	 * Atomic commit is required if we modified data on two or more
+	 * participants.
+	 */
+	if (nserverswritten <= 1)
+		return false;
+
+	ForeignTwophaseCommitIsRequired = true;
+	return true;
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_lefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		if (!fdw_part->fdwxact)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in FdwXactParticipants
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget. So we need to check if the entries are still
+		 * associated with the transaction.
+		 */
+		SpinLockAcquire(&fdwxact->mutex);
+		if (fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+			fdwxact->indoubt = true;
+			n_lefts++;
+		}
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (n_lefts > 0)
+	{
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", n_lefts);
+		FdwXactComputeRequiredXmin();
+	}
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * Wait for the foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsForeignTwophaseCommitRequested())
+		return;
+
+	/*
+	 * Also, exit if the transaction itself has no foreign transaction
+	 * participants.
+	 */
+	if (FdwXactParticipants == NIL && wait_xid == MyPgXact->xid)
+		return;
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		foreach (lcell, FdwXactParticipants)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * If the foreign transaction has FdwXact entry we might have
+			 * prepared it. Skip already-prepared foreign transaction because
+			 * it has closed its transaction. But we are not sure that foreign
+			 * transaction with status == FDWXACT_STATUS_PREPARING has been
+			 * prepared or not. So we call the rollback API to close its
+			 * transaction for safety. The prepared foreign transaction that
+			 * we might have will be resolved by the foreign transaction
+			 * resolver.
+			 */
+			if (fdw_part->fdwxact)
+			{
+				bool is_prepared;
+
+				LWLockAcquire(FdwXactLock, LW_SHARED);
+				is_prepared = fdw_part->fdwxact &&
+					fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED;
+				LWLockRelease(FdwXactLock);
+
+				if (is_prepared)
+					continue;
+			}
+
+			/* One-phase rollback foreign transaction */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, false);
+		}
+	}
+
+	/*
+	 * In commit cases, we have already prepared foreign transactions during
+	 * pre-commit phase. And these prepared transactions will be resolved by
+	 * the resolver process.
+	 */
+
+	ForgetAllFdwXactParticipants();
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Prepare foreign transactions.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (is_foreign_twophase_commit_required() &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p)
+{
+	PGPROC *proc;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+			break;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+	{
+		*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+		*waitXid_p = proc->fdwXactWaitXid;
+	}
+	else
+	{
+		*nextResolutionTs_p = -1;
+		*waitXid_p = InvalidTransactionId;
+	}
+
+	LWLockRelease(FdwXactResolutionLock);
+
+	return proc;
+}
+
+/*
+ * Get one FdwXact entry to resolve. This function intended to be used when
+ * a resolver process get FdwXact entries to resolve. So we search entries
+ * while not including in-doubt transactions and in-progress transactions.
+ */
+static FdwXact
+get_fdwxact_to_resolve(Oid dbid, TransactionId xid)
+{
+	List *fdwxacts = NIL;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Don't include both in-doubt transactions and in-progress transactions */
+	fdwxacts = get_fdwxacts(dbid, xid, InvalidOid, InvalidOid,
+							false, false, false);
+
+	return fdwxacts == NIL ? NULL : (FdwXact) linitial(fdwxacts);
+}
+
+/*
+ * Resolve one distributed transaction on the given database . The target
+ * distributed transaction is fetched from the waiting queue and its transaction
+ * participants are fetched from the global array.
+ *
+ * Release the waiter and return true after we resolved the all of the foreign
+ * transaction participants. On failure, we re-enqueue the waiting backend after
+ * incremented the next resolution time.
+ */
+void
+FdwXactResolveTransactionAndReleaseWaiter(Oid dbid, TransactionId xid,
+										  PGPROC *waiter)
+{
+	FdwXact	fdwxact;
+
+	Assert(TransactionIdIsValid(xid));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	while ((fdwxact = get_fdwxact_to_resolve(MyDatabaseId, xid)) != NULL)
+	{
+		FdwXactRslvState *state;
+		ForeignServer *server;
+		UserMapping	*usermapping;
+
+		CHECK_FOR_INTERRUPTS();
+
+		server = GetForeignServer(fdwxact->serverid);
+		usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+
+		state = create_fdwxact_state();
+		SpinLockAcquire(&fdwxact->mutex);
+		state->server = server;
+		state->usermapping = usermapping;
+		state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+		SpinLockRelease(&fdwxact->mutex);
+
+		FdwXactDetermineTransactionFate(fdwxact, false);
+
+		/* Do not hold during foreign transaction resolution */
+		LWLockRelease(FdwXactLock);
+
+		PG_TRY();
+		{
+			/*
+			 * Resolve the foreign transaction. When committing or aborting
+			 * prepared foreign transactions the previous status is always
+			 * FDWXACT_STATUS_PREPARED.
+			 */
+			FdwXactResolveForeignTransaction(fdwxact, state,
+											 FDWXACT_STATUS_PREPARED);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDWXACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				waiter->fdwXactNextResolutionTs =
+					TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+												foreign_xact_resolution_retry_interval);
+				FdwXactQueueInsert(waiter);
+			}
+			LWLockRelease(FdwXactResolutionLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved one foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter
+	 * could already be detached if user cancelled to wait before
+	 * resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId	wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/* Wake up the waiter only when we have set state and removed from queue */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Determine whether the given foreign transaction should be committed or
+ * rolled back according to the result of the local transaction. This function
+ * changes fdwxact->status so the caller must hold FdwXactLock in exclusive
+ * mode or passing need_lock with true.
+ */
+static void
+FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock)
+{
+	bool			is_commit = false;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * The being resolved transaction must be either that has been cancelled
+	 *  and marked as in-doubt or that has been prepared.
+	 */
+	Assert(fdwxact->indoubt ||
+		   fdwxact->status == FDWXACT_STATUS_PREPARED);
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	if (TransactionIdDidCommit(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_COMMITTING;
+		is_commit = true;
+	}
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+		is_commit = false;
+	}
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+		is_commit = false;
+	}
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is
+	 * about to be committed or aborted. This should not happen except for one
+	 * case where the local transaction is prepared and this foreign transaction
+	 * is being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid),
+				 errhint("The local transaction with xid %u might be prepared",
+						 fdwxact->local_xid)));
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * callback function. The 'state' is passed to the callback function. The fate of
+ * foreign transaction must be determined. If foreign transaction is resolved
+ * successfully, remove the FdwXact entry from the shared memory and also
+ * remove the corresponding on-disk file. If failed, the status of FdwXact
+ * entry changes to 'fallback_status' before erroring out.
+ */
+static void
+FdwXactResolveForeignTransaction(FdwXact fdwxact, FdwXactRslvState *state,
+								 FdwXactStatus fallback_status)
+{
+	ForeignServer		*server;
+	ForeignDataWrapper	*fdw;
+	FdwRoutine			*fdw_routine;
+	bool				is_commit;
+
+	Assert(state != NULL);
+	Assert(state->server && state->usermapping && state->fdwxact_id);
+	Assert(fdwxact != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+		elog(ERROR, "cannot resolve foreign transaction whose fate is not determined");
+
+	is_commit = fdwxact->status == FDWXACT_STATUS_COMMITTING;
+	LWLockRelease(FdwXactLock);
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	PG_TRY();
+	{
+		if (is_commit)
+			fdw_routine->CommitForeignTransaction(state);
+		else
+			fdw_routine->RollbackForeignTransaction(state);
+	}
+	PG_CATCH();
+	{
+		/* Back to the fallback status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = fallback_status;
+		LWLockRelease(FdwXactLock);
+
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	elog(DEBUG1, "successfully %s the foreign transaction with xid %u db %u server %u user %u",
+		 is_commit ? "committed" : "rolled back",
+		 fdwxact->local_xid, fdwxact->dbid, fdwxact->serverid,
+		 fdwxact->userid);
+
+	fdwxact->status = FDWXACT_STATUS_RESOLVED;
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Return palloc'd and initialized FdwXactRslvState.
+ */
+static FdwXactRslvState *
+create_fdwxact_state(void)
+{
+	FdwXactRslvState *state;
+
+	state = palloc(sizeof(FdwXactRslvState));
+	state->server = NULL;
+	state->usermapping = NULL;
+	state->fdwxact_id = NULL;
+	state->flags = 0;
+
+	return state;
+}
+
+/*
+ * Return at least one FdwXact entry that matches to given argument,
+ * otherwise return NULL. All arguments must be valid values so that it can
+ * search exactly one (or none) entry. Note that this function intended to be
+ * used for modifying the returned FdwXact entry, so the caller must hold
+ * FdwXactLock in exclusive mode and it doesn't include the in-progress
+ * FdwXact entries.
+ */
+static FdwXact
+get_one_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	/* Include in-dbout transactions but don't include in-progress ones */
+	fdwxact_list = get_fdwxacts(dbid, xid, serverid, userid,
+								true, false, false);
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdwxact_list) <= 1);
+
+	/* Could not find entry */
+	if (fdwxact_list == NIL)
+		return NULL;
+
+	return (FdwXact) linitial(fdwxact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdwxact_exists(Oid dbid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	/* Find entries from all FdwXact entries */
+	fdwxact_list = get_fdwxacts(dbid, InvalidTransactionId, serverid,
+								userid, true, true, true);
+
+	return fdwxact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_foreign_xacts, and the number of entries to num_p.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't
+ * want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdwxacts(int *num_p)
+{
+	List		*all_fdwxacts;
+	ListCell	*lc;
+	FdwXact		fdwxacts;
+	int			num_fdwxacts = 0;
+
+	Assert(num_p != NULL);
+
+	/* Get all entries */
+	all_fdwxacts = get_fdwxacts(InvalidOid, InvalidTransactionId,
+								InvalidOid, InvalidOid, true,
+								true, true);
+
+	if (all_fdwxacts == NIL)
+	{
+		*num_p = 0;
+		return NULL;
+	}
+
+	fdwxacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdwxacts));
+	*num_p = list_length(all_fdwxacts);
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdwxacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdwxacts + num_fdwxacts, fx,
+			   sizeof(FdwXactData));
+		num_fdwxacts++;
+	}
+
+	list_free(all_fdwxacts);
+
+	return fdwxacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respecitively.
+ */
+static List*
+get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			 bool include_indoubt, bool include_in_progress, bool need_lock)
+{
+	int i;
+	List	*fdwxact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact	fdwxact = FdwXactCtl->fdwxacts[i];
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* include in-doubt transaction? */
+		if (!include_indoubt && fdwxact->indoubt)
+			continue;
+
+		/* include in-progress transaction? */
+		if (!include_in_progress && FdwXactIsBeingResolved(fdwxact))
+			continue;
+
+		/* Append it if matched */
+		fdwxact_list = lappend(fdwxact_list, fdwxact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdwxact_list;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	*id;
+	int		id_len = 0;
+
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		/*
+		 * FDW doesn't provide the callback function, generate an unique
+		 * idenetifier.
+		 */
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+			 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifer \"%s\" is too long",
+						id),
+				 errdetail("foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;						/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdwxacts,
+							 serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid =
+		XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid  != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							  fdwxact_data->serverid, fdwxact_data->userid,
+							  fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know
+	 * the xact status right now. Resolver will set it later based on
+	 * the status of local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;	/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdwxact(dbid, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		return;
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(int *newval, void **extra, GucSource source)
+{
+	ForeignTwophaseCommitLevel newForeignTwophaseCommitLevel = *newval;
+
+	/* Parameter check */
+	if (newForeignTwophaseCommitLevel > FOREIGN_TWOPHASE_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	7
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdwxacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdwxacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(PG_PREPARED_FDWXACTS_COLS);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "indoubt",
+						   BOOLOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 7, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdwxacts = get_all_fdwxacts(&num_fdwxacts);
+		status->num_xacts = num_fdwxacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdwxact = &status->fdwxacts[status->cur_xact++];
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdwxact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdwxact->dbid);
+		values[1] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[2] = ObjectIdGetDatum(fdwxact->serverid);
+		values[3] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (fdwxact->status)
+		{
+			case FDWXACT_STATUS_INITIAL:
+				xact_status = "initial";
+				break;
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			case FDWXACT_STATUS_RESOLVED:
+				xact_status = "resolved";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		values[5] = BoolGetDatum(fdwxact->indoubt);
+		values[6] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	FdwXact			fdwxact;
+	FdwXactRslvState	*state;
+	FdwXactStatus		prev_status;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	server = GetForeignServer(serverid);
+	usermapping = GetUserMapping(userid, serverid);
+	state = create_fdwxact_state();
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+	{
+		LWLockRelease(FdwXactLock);
+		PG_RETURN_BOOL(false);
+	}
+
+	state->server = server;
+	state->usermapping = usermapping;
+	state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+
+	SpinLockAcquire(&fdwxact->mutex);
+	prev_status = fdwxact->status;
+	SpinLockRelease(&fdwxact->mutex);
+
+	FdwXactDetermineTransactionFate(fdwxact, false);
+
+	LWLockRelease(FdwXactLock);
+
+	FdwXactResolveForeignTransaction(fdwxact, state, prev_status);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000..45fb530
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,644 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to retry resolution.
+ */
+void
+FdwXactLauncherRequestToLaunchForRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			resolver->last_resolved_time = 0;
+			resolver->latch = NULL;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a foreign_xact_resolution_retry_interval
+		 * but always starts when the backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find FdwXact
+		 * entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int unused_slot;
+	int i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (unused_slot > max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	*resolver_dbs;	/* DBs resolver's running on */
+	HTAB	*fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL	ctl;
+	HASH_SEQ_STATUS status;
+	Oid		*entry;
+	bool	launched;
+	int		i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->indoubt)
+			continue;
+
+		hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+		return false;
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign trasaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		 /* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+						WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+						10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Returns activity of all foreign transaction resolvers.
+ */
+Datum
+pg_stat_get_foreign_xact(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000..9298877
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,344 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+bool foreign_xact_resolve_indoubt_xacts;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	MyFdwXactResolver->last_resolved_time = 0;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		PGPROC			*waiter = NULL;
+		TransactionId	waitXid = InvalidTransactionId;
+		TimestampTz		resolutionTs = -1;
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or got the waiter
+		 * that has future resolution time.
+		 */
+		while ((waiter = FdwXactGetWaiter(&resolutionTs, &waitXid)) != NULL)
+		{
+			CHECK_FOR_INTERRUPTS();
+			Assert(TransactionIdIsValid(waitXid));
+
+			if	(resolutionTs > now)
+				break;
+
+			elog(DEBUG2, "resolver got one waiter with xid %u", waitXid);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveTransactionAndReleaseWaiter(MyDatabaseId, waitXid,
+													  waiter);
+			CommitTransactionCommand();
+
+			/* Update my stats */
+			SpinLockAcquire(&(MyFdwXactResolver->mutex));
+			MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+			SpinLockRelease(&(MyFdwXactResolver->mutex));
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz last_resolved_time;
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	last_resolved_time = MyFdwXactResolver->last_resolved_time;
+	timeout = TimestampTzPlusMilliseconds(last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * checked FdwXactWaiterExists.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long	sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index 5514db1..742e825 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -8,9 +8,9 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
-	   gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
-	   mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
-	   smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdwxactdesc.o \
+	genericdesc.o  gindesc.o gistdesc.o hashdesc.o heapdesc.o \
+	logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \
+	seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 33060f3..1d4e1c8 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47..f15c83a 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -12,9 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
-	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
-	xact.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o generic_xlog.o multixact.o \
+	parallel.o rmgr.o slru.o subtrans.o timeline.o transam.o twophase.o \
+	twophase_rmgr.o varsup.o xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9368b56..8b360b1 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -9,6 +9,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
 #include "access/generic_xlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 477709b..82a0cb3 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -852,6 +853,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 }
 
 /*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
+/*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
  *
@@ -2318,6 +2348,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2377,6 +2413,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index f594d33..0d5f9d2 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1216,6 +1217,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1224,6 +1226,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1262,12 +1265,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1425,6 +1429,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2084,6 +2096,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2243,6 +2258,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2330,6 +2346,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2524,6 +2542,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2729,6 +2748,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e651a84..3716fbf 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
 #include "access/subtrans.h"
@@ -5249,6 +5250,7 @@ BootStrapXLOG(void)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6182,6 +6184,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -6704,14 +6709,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -6903,7 +6909,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7399,6 +7408,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7717,6 +7727,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -8992,6 +9005,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9425,8 +9439,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9444,6 +9460,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9460,6 +9477,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9665,6 +9683,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -9864,6 +9883,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index ea4c85e..342dd6a 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -332,6 +332,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
@@ -815,6 +818,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_foreign_xact AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_foreign_xact() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 3aeef30..43bb9ae 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2860,9 +2860,16 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
 
+	}
+
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
 
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index f96c278..621d70d 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1102,6 +1104,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1421,6 +1435,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
 	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
+	/*
 	 * Do the deletion
 	 */
 	object.classId = UserMappingRelationId;
@@ -1573,6 +1596,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index d23f292..7cdf8e1 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,7 +13,9 @@
  */
 #include "postgres.h"
 
+
 #include "access/table.h"
+#include "access/fdwxact.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
 #include "catalog/pg_inherits.h"
@@ -944,7 +946,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 52af1da..eb7450c 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,10 +226,33 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
 
+	}
+
 	return scanstate;
 }
 
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 778ff27..ed4af1a 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -38,6 +38,7 @@
 #include "postgres.h"
 
 #include "access/heapam.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
 #include "access/xact.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -550,6 +552,10 @@ ExecInsert(ModifyTableState *mtstate,
 										   NULL,
 										   specToken);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -778,6 +784,10 @@ ldelete:;
 									&tmfd,
 									changingPart);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -1325,6 +1335,10 @@ lreplace:;
 									true /* wait for commit */ ,
 									&tmfd, &lockmode, &update_indexes);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -2386,6 +2400,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index c917ec4..2780ed5 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMapping - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,20 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction &&
+		 !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction &&
+		 routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data-wrapper must support both commit and rollback routine or either");
+
+	if (routine->PrepareForeignTransaction &&
+		(!routine->CommitForeignTransaction ||
+		 !routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index b66b517..517169b 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -15,6 +15,8 @@
 #include <unistd.h>
 
 #include "libpq/pqsignal.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index d362e7f..51c3789 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3652,6 +3652,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3855,6 +3861,11 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDWXACT:
+			event_name = "FdwXact";
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -4070,6 +4081,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 62dc93d..08eb99f 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -900,6 +902,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -975,12 +981,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index 5315d93..c932167 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -154,6 +154,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index d7d7335..1491bc6 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(int port)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(int port)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 8abcfdf..b1561b2 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -91,6 +91,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -246,6 +248,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1310,6 +1313,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1375,6 +1379,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1425,6 +1430,15 @@ GetOldestXmin(Relation rel, int flags)
 		result = replication_slot_xmin;
 
 	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
+	/*
 	 * After locks have been released and vacuum_defer_cleanup_age has been
 	 * applied, check whether we need to back up further to make logical
 	 * decoding possible. We need to do so if we're computing the global limit
@@ -3014,6 +3028,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index db47843..0b8a487 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -49,3 +49,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 CLogTruncationLock					44
+FdwXactLock					45
+FdwXactResolverLock			46
+FdwXactResolutionLock			47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 498373f..dc77509 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -422,6 +423,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -823,6 +828,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index e8d8e6f..10ee130 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3007,6 +3009,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 90ffd89..7dfafe3 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -397,6 +398,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 };
 
 /*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
+/*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
  */
@@ -718,6 +738,12 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
+	/* FDWXACT_RESOLVER */
+	gettext_noop("Foreign Transaction Management / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2351,6 +2377,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FDWXACT_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FDWXACT_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4361,6 +4433,16 @@ static struct config_enum ConfigureNamesEnum[] =
 	},
 
 	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
+	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
 			NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 0fc23e3..6a3887d 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -341,6 +343,20 @@
 
 
 #------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
+#------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index 33ac627..328b857 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 88a261d..6132c72 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -210,6 +210,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index ff17804..58e9630 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -302,6 +302,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 159a30b..43144fa 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -711,6 +711,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -918,6 +919,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca..b616cea 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 3c0db2c..5798b4c 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Foreign Transactions", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index b9a531c..8238723 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 6f4013e..0cee715 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -109,6 +109,13 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 3f0de66..5c50677 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -228,6 +228,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index ff98d9e..773846d 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index cf1f409..e03b5ca 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5178,6 +5178,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '6053', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_foreign_xact', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_foreign_xact' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5891,6 +5898,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '6050', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o,o}',
+  proargnames => '{dbid,xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '6051', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '6052', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -6009,6 +6034,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '6054',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 8226860..f6592ee 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
@@ -248,7 +260,6 @@ typedef struct FdwRoutine
 	ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild;
 } FdwRoutine;
 
-
 /* Functions in foreign/foreign.c */
 extern FdwRoutine *GetFdwRoutine(Oid fdwhandler);
 extern Oid	GetForeignServerIdByRelId(Oid relid);
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 4de157c..91c2276 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -69,6 +69,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 											   bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 														 bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index fe076d8..d82d8f7 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -776,6 +776,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -853,7 +855,9 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDWXACT,
+	WAIT_EVENT_FDWXACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -933,6 +937,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index 281e1db..2eab5a9 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -21,6 +21,7 @@
 #include "storage/lock.h"
 #include "storage/pg_sema.h"
 #include "storage/proclist_types.h"
+#include "datatype/timestamp.h"
 
 /*
  * Each backend advertises up to PGPROC_MAX_CACHED_SUBXIDS TransactionIds
@@ -153,6 +154,16 @@ struct PGPROC
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
+	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
 	 * their lock.
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index da8b672..04f9c8c 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -124,4 +126,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index d68976f..d5fec50 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,9 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
+	FDWXACT_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 210e9cd..c862e0e 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1341,6 +1341,14 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.dbid,
+    f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(dbid, xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
@@ -1841,6 +1849,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_foreign_xact| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_foreign_xact() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_gssapi| SELECT s.pid,
     s.gss_auth AS gss_authenticated,
     s.gss_princ AS principal,
-- 
2.10.5

From 143280d73da137a99a4b71105f8a2a1221c850df Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Fri, 8 Feb 2019 10:44:54 +0900
Subject: [PATCH v25 1/5] Keep track of writing on non-temporary relation

---
 src/backend/executor/nodeModifyTable.c | 12 ++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 18 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 01fe11a..778ff27 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -588,6 +588,10 @@ ExecInsert(ModifyTableState *mtstate,
 							   estate->es_output_cid,
 							   0, NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -940,6 +944,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1451,6 +1459,10 @@ lreplace:;
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL, NIL);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	if (canSetTag)
 		(estate->es_processed)++;
 
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index d714551..6f4013e 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -103,6 +103,12 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
 /*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
-- 
2.10.5

From 3d157d5716121456fc54b6fea8040f6f57d9691b Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Tue, 2 Jul 2019 09:31:05 +0900
Subject: [PATCH v25 3/5] Documentation update.

---
 doc/src/sgml/catalogs.sgml                | 145 ++++++++++++++++++
 doc/src/sgml/config.sgml                  | 146 +++++++++++++++++-
 doc/src/sgml/distributed-transaction.sgml | 158 ++++++++++++++++++++
 doc/src/sgml/fdwhandler.sgml              | 236 ++++++++++++++++++++++++++++++
 doc/src/sgml/filelist.sgml                |   1 +
 doc/src/sgml/func.sgml                    |  89 +++++++++++
 doc/src/sgml/monitoring.sgml              |  60 ++++++++
 doc/src/sgml/postgres.sgml                |   1 +
 doc/src/sgml/storage.sgml                 |   6 +
 9 files changed, 841 insertions(+), 1 deletion(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 5e71a2e..1dbc45e 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8274,6 +8274,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
      </row>
 
      <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
+     <row>
       <entry><link linkend="view-pg-file-settings"><structname>pg_file_settings</structname></link></entry>
       <entry>summary of configuration file contents</entry>
      </row>
@@ -9718,6 +9723,146 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with that this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on that the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>initial</literal> : Initial status.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>resolved</literal> : This foreign transaction has been resolved.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-dbout status and
+       needs to be resolved by calling <function>pg_resolve_fdwxact</function>
+       function.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 89284dc..c1ef5d8 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4294,7 +4294,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -8611,6 +8610,151 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign
+         transaction to be resolved before the command returns a "success"
+         indication to the client. Valid values are <literal>required</literal>,
+         <literal>prefer</literal> and <literal>disabled</literal>. The default
+         setting is <literal>disabled</literal>. Setting to
+         <literal>disabled</literal> don't use two-phase commit protocol to
+         commit or rollback distributed transactions. When set to
+         <literal>required</literal> the distributed transaction strictly
+         requires that all written servers can use two-phase commit protocol.
+         That is, the distributed transaction cannot commit if even one server
+         does not support the transaction management callback routines
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. Note that when <literal>disabled</literal> or
+         <literal>prefer</literal> there can be risk of database consistency
+         among all servers that involved in the distributed transaction when some
+         foreign server crashes during committing the distributed transaction.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero
+         value to set this parameter either <literal>required</literal> or
+         <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000..350b1af
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,158 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).
+    A <productname>PostgreSQL</productname> server that received SQL is called
+    <firstterm>coordinator node</firstterm> who is responsible for coordinating
+    all the partipanting transactions. Using two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed
+    transaction manager prepares all transaction on the foreign servers if
+    two-phase commit is required. Two-phase commit is required when the
+    transaction modifies data on two or more servers including the local server
+    itself and <xref linkend="guc-foreign-twophase-commit"/>is
+    <literal>required</literal> or <literal>prefer</literal>. If all preparations
+    on foreign servers got successful go to the next step. Any failure happens
+    in this step <productname>PostgreSQL</productname> changes to rollback, then
+    rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the
+    transaction locally. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes rollback, then rollback all
+    transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolution">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that is
+    responsible for foreign transaction resolution. They commit or rollback all
+    prepared transaction on foreign servers if the coordinator received agreement
+    messages from all foreign servers during the first step.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on one database of the coordinator side. On failure during resolution, they
+    retries to resolve at an interval of
+    <varname>foreign_transaction_resolution_interval</varname> time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped. So to drop the database, you can call
+     <function>pg_stop_foreign_xact_resovler</function> function before dropping
+     the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>Manual Resolution of In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, distributed transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the coordinator node crashed during either preparing or
+    resolving distributed transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="pg-stat-foreign-xact-view"/>
+    view. These foreign transactions need to be resolved by using
+    <function>pg_resolve_foriegn_xact</function> function.
+    <productname>PostgreSQL</productname> doesn't have facilities to automatially
+    resolve in-doubt transactions. These behavior might change in a future release.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-monitoring">
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-foreign-xact-view"><literal>pg_stat_foreign_xact</literal></link>
+    view. This view contains one row for every foreign transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 27b94fb..1b292e4 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1413,6 +1413,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1892,4 +2013,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 3da2365..80a87fa 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index c878a0b..4d09fa0 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -22013,6 +22013,95 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index bf72d0c..d587164 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -368,6 +368,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_foreign_xact</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1236,6 +1244,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
+        <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
         </row>
@@ -1459,6 +1479,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
         <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
+        <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
          <entry>Waiting during base backup when throttling activity.</entry>
@@ -2338,6 +2362,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-view" xreflabel="pg_stat_foreign_xact">
+   <title><structname>pg_stat_foreign_xact</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index e59cba7..dee3f72 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -163,6 +163,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 342a0ff..840b0f6 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -84,6 +84,12 @@ Item
 </row>
 
 <row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
+<row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
 </row>
-- 
2.10.5

From c9343085d91575cb24bd52cf601ead511f102f05 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:46:01 +0900
Subject: [PATCH v25 4/5] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/Makefile                  |   7 +-
 contrib/postgres_fdw/connection.c              | 604 ++++++++++++++++---------
 contrib/postgres_fdw/expected/postgres_fdw.out | 261 ++++++++++-
 contrib/postgres_fdw/fdwxact.conf              |   3 +
 contrib/postgres_fdw/postgres_fdw.c            |  21 +-
 contrib/postgres_fdw/postgres_fdw.h            |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql      | 119 +++++
 doc/src/sgml/postgres-fdw.sgml                 |  46 ++
 8 files changed, 830 insertions(+), 238 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index 85394b4..5198f40 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -10,7 +10,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -23,3 +23,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index 57ed5f4..093eda6 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2019, PostgreSQL Global Development Group
  *
@@ -14,6 +14,7 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "catalog/pg_user_mapping.h"
@@ -56,6 +57,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -69,17 +71,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 								   SubTransactionId mySubid,
 								   SubTransactionId parentSubid,
@@ -91,24 +89,26 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 									 bool ignore_errors);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 									 PGresult **result);
-
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +128,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,12 +135,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -155,6 +148,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -182,6 +190,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +199,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +210,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,11 +226,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -414,7 +461,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -645,193 +692,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 }
 
 /*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow remote transactions that modified anything,
-					 * since it's not very reasonable to hold them open until
-					 * the prepared transaction is committed.  For the moment,
-					 * throw error unconditionally; later we might allow
-					 * read-only cases.  Note that the error will cause us to
-					 * come right back here with event == XACT_EVENT_ABORT, so
-					 * we'll clean up the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot prepare a transaction that modified remote tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
-/*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
 static void
@@ -847,10 +707,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -861,6 +717,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1195,3 +1055,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recusively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index f0c842a..067c98f 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -179,15 +198,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8781,3 +8802,225 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
+ count 
+-------
+     0
+(1 row)
+
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000..3fdbf93
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 82d8140..80de315 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include "postgres_fdw.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -503,7 +504,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 							  const PgFdwRelationInfo *fpinfo_o,
 							  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -557,6 +557,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1446,7 +1451,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2384,7 +2389,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2684,7 +2689,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3512,7 +3517,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4387,7 +4392,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4477,7 +4482,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4705,7 +4710,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index 6acb7dc..f9162bb 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "nodes/pathnodes.h"
@@ -127,7 +128,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -135,6 +136,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 							   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int	ExtractConnectionOptions(List *defelems,
@@ -201,6 +205,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 									bool is_subquery,
 									List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 630b803..572077c 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2479,3 +2502,99 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
+
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index e9ce39a..0427922 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -441,6 +441,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -469,6 +506,14 @@
   </para>
 
   <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
+  <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
    isolation level; otherwise it uses <literal>REPEATABLE READ</literal>
@@ -483,6 +528,7 @@
    COMMITTED</literal> local transaction.  A future
    <productname>PostgreSQL</productname> release might modify these rules.
   </para>
+
  </sect2>
 
  <sect2>
-- 
2.10.5

From e335c2b54eb21376f23929cf6e0b378b9275dd96 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Sun, 10 Jun 2018 18:48:08 +0900
Subject: [PATCH v25 5/5] Add regression tests for atomic commit.

---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index e66e695..b17429f 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000..9af9bb8
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index b4045ab..022ba1b 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2323,9 +2323,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2340,7 +2343,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.10.5

From a7b6737e0d29eae3954caa0ba037244abefb16b0 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Tue, 2 Jul 2019 09:32:16 +0900
Subject: [PATCH v25 2/5] Support atomic commit among multiple foreign servers.

---
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  130 ++
 src/backend/access/fdwxact/fdwxact.c          | 2827 +++++++++++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  644 ++++++
 src/backend/access/fdwxact/resolver.c         |  344 +++
 src/backend/access/rmgrdesc/Makefile          |    8 +-
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   58 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/Makefile           |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/copy.c                   |    7 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    9 +
 src/backend/executor/nodeForeignscan.c        |   25 +
 src/backend/executor/nodeModifyTable.c        |   18 +
 src/backend/foreign/foreign.c                 |   57 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   20 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   82 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  165 ++
 src/include/access/fdwxact_launcher.h         |   29 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   54 +
 src/include/access/resolver_internal.h        |   66 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   29 +
 src/include/foreign/fdwapi.h                  |   13 +-
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    9 +-
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    3 +
 src/test/regress/expected/rules.out           |   13 +
 55 files changed, 4935 insertions(+), 26 deletions(-)
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100755 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a..49480dd 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000..0207a66
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000..a6a46ad
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,130 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a conssitent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+--------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consisnts
+of the following four steps:
+
+1. Foriegn Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+we record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE all foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared tranascation on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step(commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itselft to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rolbacked the resolver process wake up the waiter.
+
+
+API Contract With Transaction Management Callback Functions
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForiegnTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for either PREPARE, COMMIT or
+ROLLBACK the trasaction on the foreign server respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transactio in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions has an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_INITIAL
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_PREPARING, FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING
+before the foreign transaction is prepared, committed and aborted by FDW
+callback functions respectively(*1). And the status then changes to
+FDWXACT_STATUS_RESOLVED once the foreign transaction are resolved, and then
+the corresponding FdwXact entry is removed with WAL logging. If failed during
+processing foreign transaction (i.g. preparing, committing or aborting) the
+status changes back to the previous status. Therefore the status
+FDWXACT_STATUS_xxxING appear only during the foreign transaction is being
+processed by an FDW callback function.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*2). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                      INVALID                       |
+ +----------------------------------------------------+
+    |                      |                       |
+    |                      v                       |
+    |           +---------------------+            |
+    |           |       INITIAL       |            |
+    |           +---------------------+            |
+   (*2)                    |                      (*2)
+    |                      v                       |
+    |           +---------------------+            |
+    |           |    PREPARING(*1)    |            |
+    |           +---------------------+            |
+    |                      |                       |
+    v                      v                       v
+ +----------------------------------------------------+
+ |                      PREPARED                      |
+ +----------------------------------------------------+
+           |                               |
+           v                               v
+ +--------------------+          +--------------------+
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |
+ +--------------------+          +--------------------+
+           |                               |
+           v                               v
+ +----------------------------------------------------+
+ |                      RESOLVED                      |
+ +----------------------------------------------------+
+
+(*1) Status that appear only during being processed by FDW
+(*2) Paths for recovered FdwXact entries
\ No newline at end of file
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100755
index 0000000..e3754e0
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2827 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers automically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit. The foreign servers are
+ * registered if FDW has both CommitForeignTransaciton API and
+ * RollbackForeignTransactionAPI. Registered participant servers are identified
+ * by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request. We have one queue
+ * of which elements are ordered by the timestamp that they expect to be
+ * processed at. Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp that they expects to be processed.
+ * Similary if failed to resolve them, it enqueues again with new timestamp
+ * (its timestamp + foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction). Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by pg_resovle_fdwxact()
+ * function.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or
+ * more servers including itself. In other case, all foreign transactions are
+ * committed or rolled back during pre-commit.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed by FDW, the corresponding
+ * FdwXact entry is update. In order to protect the entry from concurrent
+ * removing we need to hold a lock on the entry or a lock for entire global
+ * array. However, we don't want to hold the lock during FDW is processing the
+ * foreign transaction that may take a unpredictable time. To avoid this, the
+ * in-memory data of foreign transaction follows a locking model based on
+ * four linked concepts:
+ *
+ * * A foreign transaction's status variable is switched using the LWLock
+ *   FdwXactLock, which need to be hold in exclusive mode when updating the
+ *   status, while readers need to hold it in shared mode when looking at the
+ *   status.
+ * * A process who is going to update FdwXact entry cannot process foreign
+ *   transaction that is being resolved.
+ * * So setting the status to FDWACT_STATUS_PREPARING,
+ *   FDWXACT_STATUS_COMMITTING or FDWXACT_STATUS_ABORTING, which makes foreign
+ *   transaction in-progress states, means to own the FdwXact entry, which
+ *   protect it from updating/removing by concurrent writers.
+ * * Individual fields are protected by mutex where only the backend owning
+ *   the foreign transaction is authorized to update the fields from its own
+ *   one.
+
+ * Therefore, before doing PREPARE, COMMIT PREPARED or ROLLBACK PREPARED a
+ * process who is going to call transaction callback functions needs to change
+ * the status to the corresponding status above while holding FdwXactLock in
+ * exclusive mode, and call callback function after releasing the lock.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk. FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon. We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts. If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to dink.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Atomic commit is enabled by configuration */
+#define IsForeignTwophaseCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	(IsForeignTwophaseCommitEnabled() && \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED))
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define IsSeverCapableOfTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/* Check the FdwXact is begin resolved */
+#define FdwXactIsBeingResolved(fx) \
+	(((((FdwXact)(fx))->status) == FDWXACT_STATUS_PREPARING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_COMMITTING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_ABORTING))
+
+/*
+ * Structure to bundle the foreign transaction participant. This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry
+	 * is not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char			*fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool			modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact_fn;
+	CommitForeignTransaction_function	commit_foreign_xact_fn;
+	RollbackForeignTransaction_function	rollback_foreign_xact_fn;
+	GetPrepareId_function				get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit. This list
+ * has only foreign servers that provides transaction management callbacks,
+ * that is CommitForeignTransaction and RollbackForeignTransaction.
+ */
+static List *FdwXactParticipants = NIL;
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+int foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid,
+										 FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+												 bool for_commit);
+static void FdwXactResolveForeignTransaction(FdwXact fdwxact,
+											 FdwXactRslvState *state,
+											 FdwXactStatus fallback_status);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid,	void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static void FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock);
+static bool is_foreign_twophase_commit_required(void);
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static List *get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						  bool including_indoubts, bool include_in_progress,
+						  bool need_lock);
+static FdwXact get_all_fdwxacts(int *num_p);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXact get_fdwxact_to_resolve(Oid dbid, TransactionId xid);
+static FdwXactRslvState *create_fdwxact_state(void);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation		rel;
+	Oid				serverid;
+	Oid				userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign transaction identified by given server id and user id.
+ * Registered foreign transactions are managed by the global transaction
+ * manager until the end of the transaction.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant	*fdw_part;
+	ForeignServer 		*foreign_server;
+	ForeignDataWrapper	*fdw;
+	UserMapping			*user_mapping;
+	MemoryContext		old_ctx;
+	FdwRoutine			*routine;
+	ListCell	   		*lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Don't register foreign server if it doesn't provide both commit and
+	 * rollback transaction management callbacks.
+	 */
+	if (!routine->CommitForeignTransaction ||
+		!routine->RollbackForeignTransaction)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		pfree(routine);
+		return;
+	}
+
+	/*
+	 * Remember we touched the foreign server that is not capable of two-phase
+	 * commit.
+	 */
+	if (!routine->PrepareForeignTransaction)
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+	foreign_server = GetForeignServer(serverid);
+	fdw = GetForeignDataWrapper(foreign_server->fdwid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&(fdwxacts[cnt].mutex));
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * If foreign twophase commit is required, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign server's FDWs to support two-phase commit protocol and ask them to
+ *  prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and for 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_twophase_commit;
+	ListCell	*lc = NULL;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * we require all modified server have to be capable of two-phase
+	 * commit protocol.
+	 */
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false
+	 * if foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = is_foreign_twophase_commit_required();
+
+	/*
+	 * Firstly, we consider to commit foreign transactions in one-phase.
+	 */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool	commit = false;
+
+		/* Can commit in one-phase if two-phase commit is not requried */
+		if (!need_twophase_commit)
+			commit = true;
+
+		/* Non-modified foreign transaction always can be committed in one-phase */
+		if (!fdw_part->modified)
+			commit = true;
+
+		/*
+		 * In 'prefer' case, non-twophase-commit capable server can be
+		 * committed in one-phase.
+		 */
+		if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER &&
+			!IsSeverCapableOfTwophaseCommit(fdw_part))
+			commit = true;
+
+		if (commit)
+		{
+			/* Commit the foreign transaction in one-phase */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, true);
+
+			/* Delete it from the participant list */
+			FdwXactParticipants = foreach_delete_current(FdwXactParticipants,
+														 lc);
+			continue;
+		}
+	}
+
+	/* All done if we committed all foreign transactions */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * Secondary, if only one transaction is remained in the participant list
+	 * and we didn't modified the local data we can commit it without
+	 * preparation.
+	 */
+	if (list_length(FdwXactParticipants) == 1 &&
+		(MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) == 0)
+	{
+		/* Commit the foreign transaction in one-phase */
+		FdwXactOnePhaseEndForeignTransaction(linitial(FdwXactParticipants),
+											 true);
+
+		/* All foreign transaction must be committed */
+		list_free(FdwXactParticipants);
+		return;
+	}
+
+	/*
+	 * Finally, prepare foreign transactions. Note that we keep
+	 * FdwXactParticipants until the end of transaction.
+	 */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here. If any error occurs, we rollback
+ * non-prepared foreign trasactions and leave others to the resolver.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell		*lcell;
+	TransactionId	xid;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	xid = GetTopTransactionId();
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXactRslvState 	*state;
+		FdwXact		fdwxact;
+
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, xid);
+
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the FDWXACT_STATUS_PREPARING
+		 * status. Registration persists this information to the disk and logs
+		 * (that way relaying it on standby). Thus in case we loose connectivity
+		 * to the foreign server or crash ourselves, we will remember that we
+		 * might have prepared transaction on the foreign server and try to
+		 * resolve it when connectivity is restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(xid, fdw_part);
+
+		state = create_fdwxact_state();
+		state->server = fdw_part->server;
+		state->usermapping = fdw_part->usermapping;
+		state->fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+
+		/* Update the status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		Assert(fdwxact->status == FDWXACT_STATUS_INITIAL);
+		fdwxact->status = FDWXACT_STATUS_PREPARING;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 *
+		 * During abort processing, we might try to resolve a never-preapred
+		 * transaction, and get an error. This is fine as long as the FDW
+		 * provides us unique prepared transaction identifiers.
+		 */
+		PG_TRY();
+		{
+			fdw_part->prepare_foreign_xact_fn(state);
+		}
+		PG_CATCH();
+		{
+			/* failed, back to the initial state */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			fdwxact->status = FDWXACT_STATUS_INITIAL;
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		/* succeeded, update status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * One-phase commit or rollback the given foreign transaction participant.
+ */
+static void
+FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+									 bool for_commit)
+{
+	FdwXactRslvState *state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state = create_fdwxact_state();
+	state->server = fdw_part->server;
+	state->usermapping = fdw_part->usermapping;
+	state->flags = FDWXACT_FLAG_ONEPHASE;
+
+	/*
+	 * Commit or rollback foreign transaction in one-phase. Since we didn't
+	 * insert FdwXact entry for this transaction we don't need to care
+	 * failures. On failure we change to rollback.
+	 */
+	if (for_commit)
+		fdw_part->commit_foreign_xact_fn(state);
+	else
+		fdw_part->rollback_foreign_xact_fn(state);
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fdwxact;
+	FdwXactOnDiskData	*fdwxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							fdw_part->usermapping->userid,
+							fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_INITIAL;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				Oid umid, char *fdwxact_id)
+{
+	int i;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->dbid = dbid;
+	fdwxact->local_xid = xid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	fdwxact->indoubt = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (FdwXactIsBeingResolved(fdwxact))
+		elog(ERROR, "cannot remove fdwxact entry that is beging resolved");
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true and set FdwXactAtomicCommitReady to true if the current transaction
+ * modified data on two or more servers in FdwXactParticipants and
+ * local server itself.
+ */
+static bool
+is_foreign_twophase_commit_required(void)
+{
+	ListCell*	lc;
+	int			nserverswritten = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	/*
+	 * Atomic commit is required if we modified data on two or more
+	 * participants.
+	 */
+	if (nserverswritten <= 1)
+		return false;
+
+	ForeignTwophaseCommitIsRequired = true;
+	return true;
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_lefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		if (!fdw_part->fdwxact)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in FdwXactParticipants
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget. So we need to check if the entries are still
+		 * associated with the transaction.
+		 */
+		SpinLockAcquire(&fdwxact->mutex);
+		if (fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+			fdwxact->indoubt = true;
+			n_lefts++;
+		}
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (n_lefts > 0)
+	{
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", n_lefts);
+		FdwXactComputeRequiredXmin();
+	}
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * Wait for the foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsForeignTwophaseCommitRequested())
+		return;
+
+	/*
+	 * Also, exit if the transaction itself has no foreign transaction
+	 * participants.
+	 */
+	if (FdwXactParticipants == NIL && wait_xid == MyPgXact->xid)
+		return;
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		foreach (lcell, FdwXactParticipants)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * If the foreign transaction has FdwXact entry we might have
+			 * prepared it. Skip already-prepared foreign transaction because
+			 * it has closed its transaction. But we are not sure that foreign
+			 * transaction with status == FDWXACT_STATUS_PREPARING has been
+			 * prepared or not. So we call the rollback API to close its
+			 * transaction for safety. The prepared foreign transaction that
+			 * we might have will be resolved by the foreign transaction
+			 * resolver.
+			 */
+			if (fdw_part->fdwxact)
+			{
+				bool is_prepared;
+
+				LWLockAcquire(FdwXactLock, LW_SHARED);
+				is_prepared = fdw_part->fdwxact &&
+					fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED;
+				LWLockRelease(FdwXactLock);
+
+				if (is_prepared)
+					continue;
+			}
+
+			/* One-phase rollback foreign transaction */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, false);
+		}
+	}
+
+	/*
+	 * In commit cases, we have already prepared foreign transactions during
+	 * pre-commit phase. And these prepared transactions will be resolved by
+	 * the resolver process.
+	 */
+
+	ForgetAllFdwXactParticipants();
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Prepare foreign transactions.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (is_foreign_twophase_commit_required() &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p)
+{
+	PGPROC *proc;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+			break;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+	{
+		*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+		*waitXid_p = proc->fdwXactWaitXid;
+	}
+	else
+	{
+		*nextResolutionTs_p = -1;
+		*waitXid_p = InvalidTransactionId;
+	}
+
+	LWLockRelease(FdwXactResolutionLock);
+
+	return proc;
+}
+
+/*
+ * Get one FdwXact entry to resolve. This function intended to be used when
+ * a resolver process get FdwXact entries to resolve. So we search entries
+ * while not including in-doubt transactions and in-progress transactions.
+ */
+static FdwXact
+get_fdwxact_to_resolve(Oid dbid, TransactionId xid)
+{
+	List *fdwxacts = NIL;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Don't include both in-doubt transactions and in-progress transactions */
+	fdwxacts = get_fdwxacts(dbid, xid, InvalidOid, InvalidOid,
+							false, false, false);
+
+	return fdwxacts == NIL ? NULL : (FdwXact) linitial(fdwxacts);
+}
+
+/*
+ * Resolve one distributed transaction on the given database . The target
+ * distributed transaction is fetched from the waiting queue and its transaction
+ * participants are fetched from the global array.
+ *
+ * Release the waiter and return true after we resolved the all of the foreign
+ * transaction participants. On failure, we re-enqueue the waiting backend after
+ * incremented the next resolution time.
+ */
+void
+FdwXactResolveTransactionAndReleaseWaiter(Oid dbid, TransactionId xid,
+										  PGPROC *waiter)
+{
+	FdwXact	fdwxact;
+
+	Assert(TransactionIdIsValid(xid));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	while ((fdwxact = get_fdwxact_to_resolve(MyDatabaseId, xid)) != NULL)
+	{
+		FdwXactRslvState *state;
+		ForeignServer *server;
+		UserMapping	*usermapping;
+
+		CHECK_FOR_INTERRUPTS();
+
+		server = GetForeignServer(fdwxact->serverid);
+		usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+
+		state = create_fdwxact_state();
+		SpinLockAcquire(&fdwxact->mutex);
+		state->server = server;
+		state->usermapping = usermapping;
+		state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+		SpinLockRelease(&fdwxact->mutex);
+
+		FdwXactDetermineTransactionFate(fdwxact, false);
+
+		/* Do not hold during foreign transaction resolution */
+		LWLockRelease(FdwXactLock);
+
+		PG_TRY();
+		{
+			/*
+			 * Resolve the foreign transaction. When committing or aborting
+			 * prepared foreign transactions the previous status is always
+			 * FDWXACT_STATUS_PREPARED.
+			 */
+			FdwXactResolveForeignTransaction(fdwxact, state,
+											 FDWXACT_STATUS_PREPARED);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDWXACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				waiter->fdwXactNextResolutionTs =
+					TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+												foreign_xact_resolution_retry_interval);
+				FdwXactQueueInsert(waiter);
+			}
+			LWLockRelease(FdwXactResolutionLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved one foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter
+	 * could already be detached if user cancelled to wait before
+	 * resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId	wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/* Wake up the waiter only when we have set state and removed from queue */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Determine whether the given foreign transaction should be committed or
+ * rolled back according to the result of the local transaction. This function
+ * changes fdwxact->status so the caller must hold FdwXactLock in exclusive
+ * mode or passing need_lock with true.
+ */
+static void
+FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock)
+{
+	bool			is_commit = false;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * The being resolved transaction must be either that has been cancelled
+	 *  and marked as in-doubt or that has been prepared.
+	 */
+	Assert(fdwxact->indoubt ||
+		   fdwxact->status == FDWXACT_STATUS_PREPARED);
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	if (TransactionIdDidCommit(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_COMMITTING;
+		is_commit = true;
+	}
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+		is_commit = false;
+	}
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+	{
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+		is_commit = false;
+	}
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is
+	 * about to be committed or aborted. This should not happen except for one
+	 * case where the local transaction is prepared and this foreign transaction
+	 * is being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid),
+				 errhint("The local transaction with xid %u might be prepared",
+						 fdwxact->local_xid)));
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * callback function. The 'state' is passed to the callback function. The fate of
+ * foreign transaction must be determined. If foreign transaction is resolved
+ * successfully, remove the FdwXact entry from the shared memory and also
+ * remove the corresponding on-disk file. If failed, the status of FdwXact
+ * entry changes to 'fallback_status' before erroring out.
+ */
+static void
+FdwXactResolveForeignTransaction(FdwXact fdwxact, FdwXactRslvState *state,
+								 FdwXactStatus fallback_status)
+{
+	ForeignServer		*server;
+	ForeignDataWrapper	*fdw;
+	FdwRoutine			*fdw_routine;
+	bool				is_commit;
+
+	Assert(state != NULL);
+	Assert(state->server && state->usermapping && state->fdwxact_id);
+	Assert(fdwxact != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+		elog(ERROR, "cannot resolve foreign transaction whose fate is not determined");
+
+	is_commit = fdwxact->status == FDWXACT_STATUS_COMMITTING;
+	LWLockRelease(FdwXactLock);
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	PG_TRY();
+	{
+		if (is_commit)
+			fdw_routine->CommitForeignTransaction(state);
+		else
+			fdw_routine->RollbackForeignTransaction(state);
+	}
+	PG_CATCH();
+	{
+		/* Back to the fallback status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = fallback_status;
+		LWLockRelease(FdwXactLock);
+
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	elog(DEBUG1, "successfully %s the foreign transaction with xid %u db %u server %u user %u",
+		 is_commit ? "committed" : "rolled back",
+		 fdwxact->local_xid, fdwxact->dbid, fdwxact->serverid,
+		 fdwxact->userid);
+
+	fdwxact->status = FDWXACT_STATUS_RESOLVED;
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Return palloc'd and initialized FdwXactRslvState.
+ */
+static FdwXactRslvState *
+create_fdwxact_state(void)
+{
+	FdwXactRslvState *state;
+
+	state = palloc(sizeof(FdwXactRslvState));
+	state->server = NULL;
+	state->usermapping = NULL;
+	state->fdwxact_id = NULL;
+	state->flags = 0;
+
+	return state;
+}
+
+/*
+ * Return at least one FdwXact entry that matches to given argument,
+ * otherwise return NULL. All arguments must be valid values so that it can
+ * search exactly one (or none) entry. Note that this function intended to be
+ * used for modifying the returned FdwXact entry, so the caller must hold
+ * FdwXactLock in exclusive mode and it doesn't include the in-progress
+ * FdwXact entries.
+ */
+static FdwXact
+get_one_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	/* Include in-dbout transactions but don't include in-progress ones */
+	fdwxact_list = get_fdwxacts(dbid, xid, serverid, userid,
+								true, false, false);
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdwxact_list) <= 1);
+
+	/* Could not find entry */
+	if (fdwxact_list == NIL)
+		return NULL;
+
+	return (FdwXact) linitial(fdwxact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdwxact_exists(Oid dbid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	/* Find entries from all FdwXact entries */
+	fdwxact_list = get_fdwxacts(dbid, InvalidTransactionId, serverid,
+								userid, true, true, true);
+
+	return fdwxact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_foreign_xacts, and the number of entries to num_p.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't
+ * want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdwxacts(int *num_p)
+{
+	List		*all_fdwxacts;
+	ListCell	*lc;
+	FdwXact		fdwxacts;
+	int			num_fdwxacts = 0;
+
+	Assert(num_p != NULL);
+
+	/* Get all entries */
+	all_fdwxacts = get_fdwxacts(InvalidOid, InvalidTransactionId,
+								InvalidOid, InvalidOid, true,
+								true, true);
+
+	if (all_fdwxacts == NIL)
+	{
+		*num_p = 0;
+		return NULL;
+	}
+
+	fdwxacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdwxacts));
+	*num_p = list_length(all_fdwxacts);
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdwxacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdwxacts + num_fdwxacts, fx,
+			   sizeof(FdwXactData));
+		num_fdwxacts++;
+	}
+
+	list_free(all_fdwxacts);
+
+	return fdwxacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respecitively.
+ */
+static List*
+get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			 bool include_indoubt, bool include_in_progress, bool need_lock)
+{
+	int i;
+	List	*fdwxact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact	fdwxact = FdwXactCtl->fdwxacts[i];
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* include in-doubt transaction? */
+		if (!include_indoubt && fdwxact->indoubt)
+			continue;
+
+		/* include in-progress transaction? */
+		if (!include_in_progress && FdwXactIsBeingResolved(fdwxact))
+			continue;
+
+		/* Append it if matched */
+		fdwxact_list = lappend(fdwxact_list, fdwxact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdwxact_list;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	*id;
+	int		id_len = 0;
+
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		/*
+		 * FDW doesn't provide the callback function, generate an unique
+		 * idenetifier.
+		 */
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+			 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifer \"%s\" is too long",
+						id),
+				 errdetail("foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;						/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdwxacts,
+							 serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, &read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid =
+		XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid  != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							  fdwxact_data->serverid, fdwxact_data->userid,
+							  fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know
+	 * the xact status right now. Resolver will set it later based on
+	 * the status of local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;	/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdwxact(dbid, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		return;
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(int *newval, void **extra, GucSource source)
+{
+	ForeignTwophaseCommitLevel newForeignTwophaseCommitLevel = *newval;
+
+	/* Parameter check */
+	if (newForeignTwophaseCommitLevel > FOREIGN_TWOPHASE_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	7
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdwxacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdwxacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(PG_PREPARED_FDWXACTS_COLS);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "indoubt",
+						   BOOLOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 7, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdwxacts = get_all_fdwxacts(&num_fdwxacts);
+		status->num_xacts = num_fdwxacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdwxact = &status->fdwxacts[status->cur_xact++];
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdwxact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdwxact->dbid);
+		values[1] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[2] = ObjectIdGetDatum(fdwxact->serverid);
+		values[3] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (fdwxact->status)
+		{
+			case FDWXACT_STATUS_INITIAL:
+				xact_status = "initial";
+				break;
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			case FDWXACT_STATUS_RESOLVED:
+				xact_status = "resolved";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		values[5] = BoolGetDatum(fdwxact->indoubt);
+		values[6] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	FdwXact			fdwxact;
+	FdwXactRslvState	*state;
+	FdwXactStatus		prev_status;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	server = GetForeignServer(serverid);
+	usermapping = GetUserMapping(userid, serverid);
+	state = create_fdwxact_state();
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+	{
+		LWLockRelease(FdwXactLock);
+		PG_RETURN_BOOL(false);
+	}
+
+	state->server = server;
+	state->usermapping = usermapping;
+	state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+
+	SpinLockAcquire(&fdwxact->mutex);
+	prev_status = fdwxact->status;
+	SpinLockRelease(&fdwxact->mutex);
+
+	FdwXactDetermineTransactionFate(fdwxact, false);
+
+	LWLockRelease(FdwXactLock);
+
+	FdwXactResolveForeignTransaction(fdwxact, state, prev_status);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000..45fb530
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,644 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to retry resolution.
+ */
+void
+FdwXactLauncherRequestToLaunchForRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			resolver->last_resolved_time = 0;
+			resolver->latch = NULL;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a foreign_xact_resolution_retry_interval
+		 * but always starts when the backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find FdwXact
+		 * entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int unused_slot;
+	int i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (unused_slot > max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	*resolver_dbs;	/* DBs resolver's running on */
+	HTAB	*fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL	ctl;
+	HASH_SEQ_STATUS status;
+	Oid		*entry;
+	bool	launched;
+	int		i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->indoubt)
+			continue;
+
+		hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+		return false;
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign trasaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		 /* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+						WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+						10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Returns activity of all foreign transaction resolvers.
+ */
+Datum
+pg_stat_get_foreign_xact(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000..9298877
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,344 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+bool foreign_xact_resolve_indoubt_xacts;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	MyFdwXactResolver->last_resolved_time = 0;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		PGPROC			*waiter = NULL;
+		TransactionId	waitXid = InvalidTransactionId;
+		TimestampTz		resolutionTs = -1;
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or got the waiter
+		 * that has future resolution time.
+		 */
+		while ((waiter = FdwXactGetWaiter(&resolutionTs, &waitXid)) != NULL)
+		{
+			CHECK_FOR_INTERRUPTS();
+			Assert(TransactionIdIsValid(waitXid));
+
+			if	(resolutionTs > now)
+				break;
+
+			elog(DEBUG2, "resolver got one waiter with xid %u", waitXid);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveTransactionAndReleaseWaiter(MyDatabaseId, waitXid,
+													  waiter);
+			CommitTransactionCommand();
+
+			/* Update my stats */
+			SpinLockAcquire(&(MyFdwXactResolver->mutex));
+			MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+			SpinLockRelease(&(MyFdwXactResolver->mutex));
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz last_resolved_time;
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	last_resolved_time = MyFdwXactResolver->last_resolved_time;
+	timeout = TimestampTzPlusMilliseconds(last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * checked FdwXactWaiterExists.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long	sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index 5514db1..742e825 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -8,9 +8,9 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \
-	   gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \
-	   mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \
-	   smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdwxactdesc.o \
+	genericdesc.o  gindesc.o gistdesc.o hashdesc.o heapdesc.o \
+	logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \
+	seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000..fe0cef9
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxactdesc.c
+ *		PostgreSQL global transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdwxactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdwxact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		FdwXactOnDiskData *fdwxact_insert = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_insert->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_insert->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_insert->dbid);
+		appendStringInfo(buf, " local xid: %u,", fdwxact_insert->local_xid);
+		appendStringInfo(buf, " id: %s", fdwxact_insert->fdwxact_id);
+	}
+	else
+	{
+		xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_remove->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_remove->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_remove->dbid);
+		appendStringInfo(buf, " local xid: %u", fdwxact_remove->xid);
+	}
+
+}
+
+const char *
+fdwxact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDWXACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDWXACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 33060f3..1d4e1c8 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index 16fbe47..f15c83a 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -12,9 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \
-	subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
-	xact.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o generic_xlog.o multixact.o \
+	parallel.o rmgr.o slru.o subtrans.o timeline.o transam.o twophase.o \
+	twophase_rmgr.o varsup.o xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 9368b56..8b360b1 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -9,6 +9,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
 #include "access/generic_xlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 477709b..82a0cb3 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -852,6 +853,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 }
 
 /*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
+/*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
  *
@@ -2318,6 +2348,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2377,6 +2413,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index f594d33..0d5f9d2 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1216,6 +1217,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1224,6 +1226,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1262,12 +1265,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1425,6 +1429,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2084,6 +2096,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2243,6 +2258,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2330,6 +2346,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2524,6 +2542,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2729,6 +2748,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e651a84..3716fbf 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
 #include "access/subtrans.h"
@@ -5249,6 +5250,7 @@ BootStrapXLOG(void)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6182,6 +6184,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -6704,14 +6709,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -6903,7 +6909,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7399,6 +7408,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7717,6 +7727,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -8992,6 +9005,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9425,8 +9439,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9444,6 +9460,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9460,6 +9477,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9665,6 +9683,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -9864,6 +9883,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index ea4c85e..342dd6a 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -332,6 +332,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
@@ -815,6 +818,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_foreign_xact AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_foreign_xact() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 3aeef30..43bb9ae 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2860,9 +2860,16 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
 
+	}
+
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
 
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index f96c278..621d70d 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1102,6 +1104,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1421,6 +1435,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
 	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
+	/*
 	 * Do the deletion
 	 */
 	object.classId = UserMappingRelationId;
@@ -1573,6 +1596,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index d23f292..7cdf8e1 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,7 +13,9 @@
  */
 #include "postgres.h"
 
+
 #include "access/table.h"
+#include "access/fdwxact.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
 #include "catalog/pg_inherits.h"
@@ -944,7 +946,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 52af1da..eb7450c 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,10 +226,33 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
 
+	}
+
 	return scanstate;
 }
 
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 778ff27..ed4af1a 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -38,6 +38,7 @@
 #include "postgres.h"
 
 #include "access/heapam.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
 #include "access/xact.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -550,6 +552,10 @@ ExecInsert(ModifyTableState *mtstate,
 										   NULL,
 										   specToken);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -778,6 +784,10 @@ ldelete:;
 									&tmfd,
 									changingPart);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -1325,6 +1335,10 @@ lreplace:;
 									true /* wait for commit */ ,
 									&tmfd, &lockmode, &update_indexes);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -2386,6 +2400,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index c917ec4..2780ed5 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMapping - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,20 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction &&
+		 !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction &&
+		 routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data-wrapper must support both commit and rollback routine or either");
+
+	if (routine->PrepareForeignTransaction &&
+		(!routine->CommitForeignTransaction ||
+		 !routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index b66b517..517169b 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -15,6 +15,8 @@
 #include <unistd.h>
 
 #include "libpq/pqsignal.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index d362e7f..51c3789 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3652,6 +3652,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3855,6 +3861,11 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDWXACT:
+			event_name = "FdwXact";
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -4070,6 +4081,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 62dc93d..08eb99f 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -900,6 +902,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -975,12 +981,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index 5315d93..c932167 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -154,6 +154,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index d7d7335..1491bc6 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(int port)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(int port)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 8abcfdf..b1561b2 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -91,6 +91,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -246,6 +248,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1310,6 +1313,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1375,6 +1379,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1425,6 +1430,15 @@ GetOldestXmin(Relation rel, int flags)
 		result = replication_slot_xmin;
 
 	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
+	/*
 	 * After locks have been released and vacuum_defer_cleanup_age has been
 	 * applied, check whether we need to back up further to make logical
 	 * decoding possible. We need to do so if we're computing the global limit
@@ -3014,6 +3028,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index db47843..0b8a487 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -49,3 +49,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 CLogTruncationLock					44
+FdwXactLock					45
+FdwXactResolverLock			46
+FdwXactResolutionLock			47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 498373f..dc77509 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -422,6 +423,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -823,6 +828,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index e8d8e6f..10ee130 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3007,6 +3009,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 90ffd89..7dfafe3 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -397,6 +398,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 };
 
 /*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
+/*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
  */
@@ -718,6 +738,12 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
+	/* FDWXACT_RESOLVER */
+	gettext_noop("Foreign Transaction Management / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2351,6 +2377,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FDWXACT_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FDWXACT_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4361,6 +4433,16 @@ static struct config_enum ConfigureNamesEnum[] =
 	},
 
 	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
+	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
 			NULL
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 0fc23e3..6a3887d 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -341,6 +343,20 @@
 
 
 #------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
+#------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
 
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index 33ac627..328b857 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 88a261d..6132c72 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -210,6 +210,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index ff17804..58e9630 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -302,6 +302,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 159a30b..43144fa 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -711,6 +711,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -918,6 +919,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca..b616cea 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000..147d41c
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,165 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL global transaction manager
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDWXACT_H
+#define FDWXACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* fdwXactState */
+#define	FDWXACT_NOT_WAITING		0
+#define	FDWXACT_WAITING			1
+#define	FDWXACT_WAIT_COMPLETE	2
+
+/* Flag passed to FDW transaction management APIs */
+#define FDWXACT_FLAG_ONEPHASE		0x01	/* transaction can commit/rollback
+											   without preparation */
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER,		/* use twophase commit where available */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support
+										   twophase commit */
+} ForeignTwophaseCommitLevel;
+
+/* Enum to track the status of foreign transaction */
+typedef enum
+{
+	FDWXACT_STATUS_INVALID,
+	FDWXACT_STATUS_INITIAL,
+	FDWXACT_STATUS_PREPARING,		/* foreign transaction is being prepared */
+	FDWXACT_STATUS_PREPARED,		/* foreign transaction is prepared */
+	FDWXACT_STATUS_COMMITTING,		/* foreign prepared transaction is to
+									 * be committed */
+	FDWXACT_STATUS_ABORTING,		/* foreign prepared transaction is to be
+									 * aborted */
+	FDWXACT_STATUS_RESOLVED
+} FdwXactStatus;
+
+typedef struct FdwXactData *FdwXact;
+
+/*
+ * Shared memory state of a single foreign transaction.
+ */
+typedef struct FdwXactData
+{
+	FdwXact			fdwxact_free_next;	/* Next free FdwXact entry */
+
+	Oid				dbid;			/* database oid where to find foreign server
+									 * and user mapping */
+	TransactionId	local_xid;		/* XID of local transaction */
+	Oid				serverid;		/* foreign server where transaction takes
+									 * place */
+	Oid				userid;			/* user who initiated the foreign
+									 * transaction */
+	Oid				umid;
+	bool			indoubt;		/* Is an in-doubt transaction? */
+	slock_t			mutex;			/* Protect the above fields */
+
+	/* The status of the foreign transaction, protected by FdwXactLock */
+	FdwXactStatus 	status;
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;		/* XLOG offset of inserting this entry start */
+	XLogRecPtr	insert_end_lsn;		/* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN];		/* prepared transaction identifier */
+} FdwXactData;
+
+/*
+ * Shared memory layout for maintaining foreign prepared transaction entries.
+ * Adding or removing FdwXact entry needs to hold FdwXactLock in exclusive mode,
+ * and iterating fdwXacts needs that in shared mode.
+ */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		free_fdwxacts;
+
+	/* Number of valid foreign transaction entries */
+	int			num_fdwxacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdwxacts[FLEXIBLE_ARRAY_MEMBER];		/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* State data for foreign transaction resolution, passed to FDW callbacks */
+typedef struct FdwXactRslvState
+{
+	/* Foreign transaction information */
+	char	*fdwxact_id;
+
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	int		flags;			/* OR of FDWXACT_FLAG_xx flags */
+} FdwXactRslvState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+/* Function declarations */
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern bool fdwxact_exists(Oid dboid, Oid serverid, Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern void PreCommit_FdwXacts(void);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+extern void FdwXactResolveTransactionAndReleaseWaiter(Oid dbid, TransactionId xid,
+													  PGPROC *waiter);
+extern bool FdwXactResolveInDoubtTransactions(Oid dbid);
+extern PGPROC *FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p);
+extern void FdwXactCleanupAtProcExit(void);
+extern void RegisterFdwXactByRelId(Oid relid, bool modified);
+extern void RegisterFdwXactByServerId(Oid serverid, bool modified);
+extern void FdwXactMarkForeignServerAccessed(Oid relid, bool modified);
+extern bool check_foreign_twophase_commit(int *newval, void **extra,
+										  GucSource source);
+extern bool FdwXactWaiterExists(Oid dbid);
+
+#endif   /* FDWXACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000..dd0f5d1
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,29 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherRequestToLaunch(void);
+extern void FdwXactLauncherRequestToLaunchForRetry(void);
+extern void FdwXactLaunchOrWakeupResolver(void);
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+
+
+#endif	/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000..2607654
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int foreign_xact_resolver_timeout;
+
+#endif		/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000..39ca66b
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,54 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDWXACT_INSERT	0x00
+#define XLOG_FDWXACT_REMOVE	0x10
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDWXACT_ID_MAX_LEN 200
+
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdwxact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+	bool		force;
+} xl_fdwxact_remove;
+
+extern void fdwxact_redo(XLogReaderState *record);
+extern void fdwxact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdwxact_identify(uint8 info);
+
+#endif	/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000..55fc970
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,66 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/resovler_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t	pid;	/* this resolver's PID, or 0 if not active */
+	Oid		dbid;	/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool	in_use;
+
+	/* Stats */
+	TimestampTz	last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t	mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	*latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/* Foreign transaction resolution queue. Protected by FdwXactLock */
+	SHM_QUEUE	fdwxact_queue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch		*launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+#define SizeOfFdwXactRslvCtlData \
+	(offsetof(FdwXactRslvCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif	/* RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 3c0db2c..5798b4c 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Foreign Transactions", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index b9a531c..8238723 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 6f4013e..0cee715 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -109,6 +109,13 @@ extern int	MyXactFlags;
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
 /*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
+/*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
 typedef enum
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 3f0de66..5c50677 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -228,6 +228,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index ff98d9e..773846d 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index cf1f409..e03b5ca 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5178,6 +5178,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '6053', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_foreign_xact', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_foreign_xact' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5891,6 +5898,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '6050', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o,o}',
+  proargnames => '{dbid,xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '6051', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '6052', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -6009,6 +6034,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '6054',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 8226860..f6592ee 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
@@ -248,7 +260,6 @@ typedef struct FdwRoutine
 	ReparameterizeForeignPathByChild_function ReparameterizeForeignPathByChild;
 } FdwRoutine;
 
-
 /* Functions in foreign/foreign.c */
 extern FdwRoutine *GetFdwRoutine(Oid fdwhandler);
 extern Oid	GetForeignServerIdByRelId(Oid relid);
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 4de157c..91c2276 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -69,6 +69,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 											   bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 														 bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index fe076d8..d82d8f7 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -776,6 +776,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -853,7 +855,9 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDWXACT,
+	WAIT_EVENT_FDWXACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -933,6 +937,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index 281e1db..2eab5a9 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -21,6 +21,7 @@
 #include "storage/lock.h"
 #include "storage/pg_sema.h"
 #include "storage/proclist_types.h"
+#include "datatype/timestamp.h"
 
 /*
  * Each backend advertises up to PGPROC_MAX_CACHED_SUBXIDS TransactionIds
@@ -153,6 +154,16 @@ struct PGPROC
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
 	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
+	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
 	 * their lock.
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index da8b672..04f9c8c 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -124,4 +126,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index d68976f..d5fec50 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,9 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
+	FDWXACT_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 210e9cd..c862e0e 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1341,6 +1341,14 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.dbid,
+    f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(dbid, xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
@@ -1841,6 +1849,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_foreign_xact| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_foreign_xact() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_gssapi| SELECT s.pid,
     s.gss_auth AS gss_authenticated,
     s.gss_princ AS principal,
-- 
2.10.5

From 733f1e413ef2b2fe1d3ecba41eb4cd8e355ab826 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 5 Dec 2019 16:59:47 +0900
Subject: [PATCH v26 1/5] Keep track of writing on non-temporary relation

Original Author: Masahiko Sawada <sawada.mshk@gmail.com>
---
 src/backend/executor/nodeModifyTable.c | 12 ++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 18 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index e3eb9d7b90..cd91f9c8a8 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -587,6 +587,10 @@ ExecInsert(ModifyTableState *mtstate,
 							   estate->es_output_cid,
 							   0, NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -938,6 +942,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1447,6 +1455,10 @@ lreplace:;
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL, NIL);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	if (canSetTag)
 		(estate->es_processed)++;
 
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 9d2899dea1..cb5c4935d2 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -102,6 +102,12 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
-- 
2.23.0

From d21c72a7db85c2211504f60fca8d39c0bd0ee5a6 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 5 Dec 2019 17:00:50 +0900
Subject: [PATCH v26 2/5] Support atomic commit among multiple foreign servers.

Original Author: Masahiko Sawada <sawada.mshk@gmail.com>
---
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  130 +
 src/backend/access/fdwxact/fdwxact.c          | 2816 +++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  644 ++++
 src/backend/access/fdwxact/resolver.c         |  344 ++
 src/backend/access/rmgrdesc/Makefile          |    1 +
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   58 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   27 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/copy.c                   |    6 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    8 +
 src/backend/executor/nodeForeignscan.c        |   24 +
 src/backend/executor/nodeModifyTable.c        |   18 +
 src/backend/foreign/foreign.c                 |   57 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   20 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   82 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/fdwxactdesc.c              |    1 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  165 +
 src/include/access/fdwxact_launcher.h         |   29 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   54 +
 src/include/access/resolver_internal.h        |   66 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   29 +
 src/include/foreign/fdwapi.h                  |   12 +
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    9 +-
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    3 +
 src/test/regress/expected/rules.out           |   13 +
 55 files changed, 4917 insertions(+), 18 deletions(-)
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100644 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 120000 src/bin/pg_waldump/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a8bb..49480dd039 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000000..0207a66fb4
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000000..46ccb7eeae
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,130 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a conssitent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+--------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consisnts
+of the following four steps:
+
+1. Foriegn Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+we record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE all foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared tranascation on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step(commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itselft to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rolbacked the resolver process wake up the waiter.
+
+
+API Contract With Transaction Management Callback Functions
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForiegnTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for either PREPARE, COMMIT or
+ROLLBACK the trasaction on the foreign server respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transactio in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions has an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_INITIAL
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_PREPARING, FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING
+before the foreign transaction is prepared, committed and aborted by FDW
+callback functions respectively(*1). And the status then changes to
+FDWXACT_STATUS_RESOLVED once the foreign transaction are resolved, and then
+the corresponding FdwXact entry is removed with WAL logging. If failed during
+processing foreign transaction (i.g. preparing, committing or aborting) the
+status changes back to the previous status. Therefore the status
+FDWXACT_STATUS_xxxING appear only during the foreign transaction is being
+processed by an FDW callback function.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*2). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                      INVALID                       |
+ +----------------------------------------------------+
+    |                      |                       |
+    |                      v                       |
+    |           +---------------------+            |
+    |           |       INITIAL       |            |
+    |           +---------------------+            |
+   (*2)                    |                      (*2)
+    |                      v                       |
+    |           +---------------------+            |
+    |           |    PREPARING(*1)    |            |
+    |           +---------------------+            |
+    |                      |                       |
+    v                      v                       v
+ +----------------------------------------------------+
+ |                      PREPARED                      |
+ +----------------------------------------------------+
+           |                               |
+           v                               v
+ +--------------------+          +--------------------+
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |
+ +--------------------+          +--------------------+
+           |                               |
+           v                               v
+ +----------------------------------------------------+
+ |                      RESOLVED                      |
+ +----------------------------------------------------+
+
+(*1) Status that appear only during being processed by FDW
+(*2) Paths for recovered FdwXact entries
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100644
index 0000000000..058a416f81
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2816 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers automically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit. The foreign servers are
+ * registered if FDW has both CommitForeignTransaciton API and
+ * RollbackForeignTransactionAPI. Registered participant servers are identified
+ * by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request. We have one queue
+ * of which elements are ordered by the timestamp that they expect to be
+ * processed at. Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp that they expects to be processed.
+ * Similary if failed to resolve them, it enqueues again with new timestamp
+ * (its timestamp + foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction). Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by pg_resovle_fdwxact()
+ * function.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or
+ * more servers including itself. In other case, all foreign transactions are
+ * committed or rolled back during pre-commit.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed by FDW, the corresponding
+ * FdwXact entry is update. In order to protect the entry from concurrent
+ * removing we need to hold a lock on the entry or a lock for entire global
+ * array. However, we don't want to hold the lock during FDW is processing the
+ * foreign transaction that may take a unpredictable time. To avoid this, the
+ * in-memory data of foreign transaction follows a locking model based on
+ * four linked concepts:
+ *
+ * * A foreign transaction's status variable is switched using the LWLock
+ *   FdwXactLock, which need to be hold in exclusive mode when updating the
+ *   status, while readers need to hold it in shared mode when looking at the
+ *   status.
+ * * A process who is going to update FdwXact entry cannot process foreign
+ *   transaction that is being resolved.
+ * * So setting the status to FDWACT_STATUS_PREPARING,
+ *   FDWXACT_STATUS_COMMITTING or FDWXACT_STATUS_ABORTING, which makes foreign
+ *   transaction in-progress states, means to own the FdwXact entry, which
+ *   protect it from updating/removing by concurrent writers.
+ * * Individual fields are protected by mutex where only the backend owning
+ *   the foreign transaction is authorized to update the fields from its own
+ *   one.
+
+ * Therefore, before doing PREPARE, COMMIT PREPARED or ROLLBACK PREPARED a
+ * process who is going to call transaction callback functions needs to change
+ * the status to the corresponding status above while holding FdwXactLock in
+ * exclusive mode, and call callback function after releasing the lock.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk. FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon. We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts. If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to dink.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Atomic commit is enabled by configuration */
+#define IsForeignTwophaseCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	(IsForeignTwophaseCommitEnabled() && \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED))
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define IsSeverCapableOfTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/* Check the FdwXact is begin resolved */
+#define FdwXactIsBeingResolved(fx) \
+	(((((FdwXact)(fx))->status) == FDWXACT_STATUS_PREPARING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_COMMITTING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_ABORTING))
+
+/*
+ * Structure to bundle the foreign transaction participant. This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry
+	 * is not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char			*fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool			modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact_fn;
+	CommitForeignTransaction_function	commit_foreign_xact_fn;
+	RollbackForeignTransaction_function	rollback_foreign_xact_fn;
+	GetPrepareId_function				get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit. This list
+ * has only foreign servers that provides transaction management callbacks,
+ * that is CommitForeignTransaction and RollbackForeignTransaction.
+ */
+static List *FdwXactParticipants = NIL;
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+int foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid,
+										 FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+												 bool for_commit);
+static void FdwXactResolveForeignTransaction(FdwXact fdwxact,
+											 FdwXactRslvState *state,
+											 FdwXactStatus fallback_status);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid,	void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static void FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock);
+static bool is_foreign_twophase_commit_required(void);
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static List *get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						  bool including_indoubts, bool include_in_progress,
+						  bool need_lock);
+static FdwXact get_all_fdwxacts(int *num_p);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXact get_fdwxact_to_resolve(Oid dbid, TransactionId xid);
+static FdwXactRslvState *create_fdwxact_state(void);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation		rel;
+	Oid				serverid;
+	Oid				userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign transaction identified by given server id and user id.
+ * Registered foreign transactions are managed by the global transaction
+ * manager until the end of the transaction.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant	*fdw_part;
+	ForeignServer 		*foreign_server;
+	UserMapping			*user_mapping;
+	MemoryContext		old_ctx;
+	FdwRoutine			*routine;
+	ListCell	   		*lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Don't register foreign server if it doesn't provide both commit and
+	 * rollback transaction management callbacks.
+	 */
+	if (!routine->CommitForeignTransaction ||
+		!routine->RollbackForeignTransaction)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		pfree(routine);
+		return;
+	}
+
+	/*
+	 * Remember we touched the foreign server that is not capable of two-phase
+	 * commit.
+	 */
+	if (!routine->PrepareForeignTransaction)
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+	foreign_server = GetForeignServer(serverid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&(fdwxacts[cnt].mutex));
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * If foreign twophase commit is required, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign server's FDWs to support two-phase commit protocol and ask them to
+ *  prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and for 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_twophase_commit;
+	ListCell	*lc = NULL;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * we require all modified server have to be capable of two-phase
+	 * commit protocol.
+	 */
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false
+	 * if foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = is_foreign_twophase_commit_required();
+
+	/*
+	 * Firstly, we consider to commit foreign transactions in one-phase.
+	 */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool	commit = false;
+
+		/* Can commit in one-phase if two-phase commit is not requried */
+		if (!need_twophase_commit)
+			commit = true;
+
+		/* Non-modified foreign transaction always can be committed in one-phase */
+		if (!fdw_part->modified)
+			commit = true;
+
+		/*
+		 * In 'prefer' case, non-twophase-commit capable server can be
+		 * committed in one-phase.
+		 */
+		if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER &&
+			!IsSeverCapableOfTwophaseCommit(fdw_part))
+			commit = true;
+
+		if (commit)
+		{
+			/* Commit the foreign transaction in one-phase */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, true);
+
+			/* Delete it from the participant list */
+			FdwXactParticipants = foreach_delete_current(FdwXactParticipants,
+														 lc);
+			continue;
+		}
+	}
+
+	/* All done if we committed all foreign transactions */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * Secondary, if only one transaction is remained in the participant list
+	 * and we didn't modified the local data we can commit it without
+	 * preparation.
+	 */
+	if (list_length(FdwXactParticipants) == 1 &&
+		(MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) == 0)
+	{
+		/* Commit the foreign transaction in one-phase */
+		FdwXactOnePhaseEndForeignTransaction(linitial(FdwXactParticipants),
+											 true);
+
+		/* All foreign transaction must be committed */
+		list_free(FdwXactParticipants);
+		return;
+	}
+
+	/*
+	 * Finally, prepare foreign transactions. Note that we keep
+	 * FdwXactParticipants until the end of transaction.
+	 */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here. If any error occurs, we rollback
+ * non-prepared foreign trasactions and leave others to the resolver.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell		*lcell;
+	TransactionId	xid;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	xid = GetTopTransactionId();
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXactRslvState 	*state;
+		FdwXact		fdwxact;
+
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, xid);
+
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the FDWXACT_STATUS_PREPARING
+		 * status. Registration persists this information to the disk and logs
+		 * (that way relaying it on standby). Thus in case we loose connectivity
+		 * to the foreign server or crash ourselves, we will remember that we
+		 * might have prepared transaction on the foreign server and try to
+		 * resolve it when connectivity is restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(xid, fdw_part);
+
+		state = create_fdwxact_state();
+		state->server = fdw_part->server;
+		state->usermapping = fdw_part->usermapping;
+		state->fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+
+		/* Update the status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		Assert(fdwxact->status == FDWXACT_STATUS_INITIAL);
+		fdwxact->status = FDWXACT_STATUS_PREPARING;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 *
+		 * During abort processing, we might try to resolve a never-preapred
+		 * transaction, and get an error. This is fine as long as the FDW
+		 * provides us unique prepared transaction identifiers.
+		 */
+		PG_TRY();
+		{
+			fdw_part->prepare_foreign_xact_fn(state);
+		}
+		PG_CATCH();
+		{
+			/* failed, back to the initial state */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			fdwxact->status = FDWXACT_STATUS_INITIAL;
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		/* succeeded, update status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * One-phase commit or rollback the given foreign transaction participant.
+ */
+static void
+FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+									 bool for_commit)
+{
+	FdwXactRslvState *state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state = create_fdwxact_state();
+	state->server = fdw_part->server;
+	state->usermapping = fdw_part->usermapping;
+	state->flags = FDWXACT_FLAG_ONEPHASE;
+
+	/*
+	 * Commit or rollback foreign transaction in one-phase. Since we didn't
+	 * insert FdwXact entry for this transaction we don't need to care
+	 * failures. On failure we change to rollback.
+	 */
+	if (for_commit)
+		fdw_part->commit_foreign_xact_fn(state);
+	else
+		fdw_part->rollback_foreign_xact_fn(state);
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fdwxact;
+	FdwXactOnDiskData	*fdwxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							fdw_part->usermapping->userid,
+							fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_INITIAL;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				Oid umid, char *fdwxact_id)
+{
+	int i;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->dbid = dbid;
+	fdwxact->local_xid = xid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	fdwxact->indoubt = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (FdwXactIsBeingResolved(fdwxact))
+		elog(ERROR, "cannot remove fdwxact entry that is beging resolved");
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true and set FdwXactAtomicCommitReady to true if the current transaction
+ * modified data on two or more servers in FdwXactParticipants and
+ * local server itself.
+ */
+static bool
+is_foreign_twophase_commit_required(void)
+{
+	ListCell*	lc;
+	int			nserverswritten = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	/*
+	 * Atomic commit is required if we modified data on two or more
+	 * participants.
+	 */
+	if (nserverswritten <= 1)
+		return false;
+
+	ForeignTwophaseCommitIsRequired = true;
+	return true;
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_lefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		if (!fdw_part->fdwxact)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in FdwXactParticipants
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget. So we need to check if the entries are still
+		 * associated with the transaction.
+		 */
+		SpinLockAcquire(&fdwxact->mutex);
+		if (fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+			fdwxact->indoubt = true;
+			n_lefts++;
+		}
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (n_lefts > 0)
+	{
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", n_lefts);
+		FdwXactComputeRequiredXmin();
+	}
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * Wait for the foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsForeignTwophaseCommitRequested())
+		return;
+
+	/*
+	 * Also, exit if the transaction itself has no foreign transaction
+	 * participants.
+	 */
+	if (FdwXactParticipants == NIL && wait_xid == MyPgXact->xid)
+		return;
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		foreach (lcell, FdwXactParticipants)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * If the foreign transaction has FdwXact entry we might have
+			 * prepared it. Skip already-prepared foreign transaction because
+			 * it has closed its transaction. But we are not sure that foreign
+			 * transaction with status == FDWXACT_STATUS_PREPARING has been
+			 * prepared or not. So we call the rollback API to close its
+			 * transaction for safety. The prepared foreign transaction that
+			 * we might have will be resolved by the foreign transaction
+			 * resolver.
+			 */
+			if (fdw_part->fdwxact)
+			{
+				bool is_prepared;
+
+				LWLockAcquire(FdwXactLock, LW_SHARED);
+				is_prepared = fdw_part->fdwxact &&
+					fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED;
+				LWLockRelease(FdwXactLock);
+
+				if (is_prepared)
+					continue;
+			}
+
+			/* One-phase rollback foreign transaction */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, false);
+		}
+	}
+
+	/*
+	 * In commit cases, we have already prepared foreign transactions during
+	 * pre-commit phase. And these prepared transactions will be resolved by
+	 * the resolver process.
+	 */
+
+	ForgetAllFdwXactParticipants();
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Prepare foreign transactions.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (is_foreign_twophase_commit_required() &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p)
+{
+	PGPROC *proc;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+			break;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+	{
+		*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+		*waitXid_p = proc->fdwXactWaitXid;
+	}
+	else
+	{
+		*nextResolutionTs_p = -1;
+		*waitXid_p = InvalidTransactionId;
+	}
+
+	LWLockRelease(FdwXactResolutionLock);
+
+	return proc;
+}
+
+/*
+ * Get one FdwXact entry to resolve. This function intended to be used when
+ * a resolver process get FdwXact entries to resolve. So we search entries
+ * while not including in-doubt transactions and in-progress transactions.
+ */
+static FdwXact
+get_fdwxact_to_resolve(Oid dbid, TransactionId xid)
+{
+	List *fdwxacts = NIL;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Don't include both in-doubt transactions and in-progress transactions */
+	fdwxacts = get_fdwxacts(dbid, xid, InvalidOid, InvalidOid,
+							false, false, false);
+
+	return fdwxacts == NIL ? NULL : (FdwXact) linitial(fdwxacts);
+}
+
+/*
+ * Resolve one distributed transaction on the given database . The target
+ * distributed transaction is fetched from the waiting queue and its transaction
+ * participants are fetched from the global array.
+ *
+ * Release the waiter and return true after we resolved the all of the foreign
+ * transaction participants. On failure, we re-enqueue the waiting backend after
+ * incremented the next resolution time.
+ */
+void
+FdwXactResolveTransactionAndReleaseWaiter(Oid dbid, TransactionId xid,
+										  PGPROC *waiter)
+{
+	FdwXact	fdwxact;
+
+	Assert(TransactionIdIsValid(xid));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	while ((fdwxact = get_fdwxact_to_resolve(MyDatabaseId, xid)) != NULL)
+	{
+		FdwXactRslvState *state;
+		ForeignServer *server;
+		UserMapping	*usermapping;
+
+		CHECK_FOR_INTERRUPTS();
+
+		server = GetForeignServer(fdwxact->serverid);
+		usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+
+		state = create_fdwxact_state();
+		SpinLockAcquire(&fdwxact->mutex);
+		state->server = server;
+		state->usermapping = usermapping;
+		state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+		SpinLockRelease(&fdwxact->mutex);
+
+		FdwXactDetermineTransactionFate(fdwxact, false);
+
+		/* Do not hold during foreign transaction resolution */
+		LWLockRelease(FdwXactLock);
+
+		PG_TRY();
+		{
+			/*
+			 * Resolve the foreign transaction. When committing or aborting
+			 * prepared foreign transactions the previous status is always
+			 * FDWXACT_STATUS_PREPARED.
+			 */
+			FdwXactResolveForeignTransaction(fdwxact, state,
+											 FDWXACT_STATUS_PREPARED);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDWXACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				waiter->fdwXactNextResolutionTs =
+					TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+												foreign_xact_resolution_retry_interval);
+				FdwXactQueueInsert(waiter);
+			}
+			LWLockRelease(FdwXactResolutionLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved one foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter
+	 * could already be detached if user cancelled to wait before
+	 * resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId	wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/* Wake up the waiter only when we have set state and removed from queue */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Determine whether the given foreign transaction should be committed or
+ * rolled back according to the result of the local transaction. This function
+ * changes fdwxact->status so the caller must hold FdwXactLock in exclusive
+ * mode or passing need_lock with true.
+ */
+static void
+FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock)
+{
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * The being resolved transaction must be either that has been cancelled
+	 *  and marked as in-doubt or that has been prepared.
+	 */
+	Assert(fdwxact->indoubt ||
+		   fdwxact->status == FDWXACT_STATUS_PREPARED);
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	if (TransactionIdDidCommit(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_COMMITTING;
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is
+	 * about to be committed or aborted. This should not happen except for one
+	 * case where the local transaction is prepared and this foreign transaction
+	 * is being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid),
+				 errhint("The local transaction with xid %u might be prepared",
+						 fdwxact->local_xid)));
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * callback function. The 'state' is passed to the callback function. The fate of
+ * foreign transaction must be determined. If foreign transaction is resolved
+ * successfully, remove the FdwXact entry from the shared memory and also
+ * remove the corresponding on-disk file. If failed, the status of FdwXact
+ * entry changes to 'fallback_status' before erroring out.
+ */
+static void
+FdwXactResolveForeignTransaction(FdwXact fdwxact, FdwXactRslvState *state,
+								 FdwXactStatus fallback_status)
+{
+	ForeignServer		*server;
+	ForeignDataWrapper	*fdw;
+	FdwRoutine			*fdw_routine;
+	bool				is_commit;
+
+	Assert(state != NULL);
+	Assert(state->server && state->usermapping && state->fdwxact_id);
+	Assert(fdwxact != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+		elog(ERROR, "cannot resolve foreign transaction whose fate is not determined");
+
+	is_commit = fdwxact->status == FDWXACT_STATUS_COMMITTING;
+	LWLockRelease(FdwXactLock);
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	PG_TRY();
+	{
+		if (is_commit)
+			fdw_routine->CommitForeignTransaction(state);
+		else
+			fdw_routine->RollbackForeignTransaction(state);
+	}
+	PG_CATCH();
+	{
+		/* Back to the fallback status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = fallback_status;
+		LWLockRelease(FdwXactLock);
+
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	elog(DEBUG1, "successfully %s the foreign transaction with xid %u db %u server %u user %u",
+		 is_commit ? "committed" : "rolled back",
+		 fdwxact->local_xid, fdwxact->dbid, fdwxact->serverid,
+		 fdwxact->userid);
+
+	fdwxact->status = FDWXACT_STATUS_RESOLVED;
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Return palloc'd and initialized FdwXactRslvState.
+ */
+static FdwXactRslvState *
+create_fdwxact_state(void)
+{
+	FdwXactRslvState *state;
+
+	state = palloc(sizeof(FdwXactRslvState));
+	state->server = NULL;
+	state->usermapping = NULL;
+	state->fdwxact_id = NULL;
+	state->flags = 0;
+
+	return state;
+}
+
+/*
+ * Return at least one FdwXact entry that matches to given argument,
+ * otherwise return NULL. All arguments must be valid values so that it can
+ * search exactly one (or none) entry. Note that this function intended to be
+ * used for modifying the returned FdwXact entry, so the caller must hold
+ * FdwXactLock in exclusive mode and it doesn't include the in-progress
+ * FdwXact entries.
+ */
+static FdwXact
+get_one_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	/* Include in-dbout transactions but don't include in-progress ones */
+	fdwxact_list = get_fdwxacts(dbid, xid, serverid, userid,
+								true, false, false);
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdwxact_list) <= 1);
+
+	/* Could not find entry */
+	if (fdwxact_list == NIL)
+		return NULL;
+
+	return (FdwXact) linitial(fdwxact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdwxact_exists(Oid dbid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	/* Find entries from all FdwXact entries */
+	fdwxact_list = get_fdwxacts(dbid, InvalidTransactionId, serverid,
+								userid, true, true, true);
+
+	return fdwxact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_foreign_xacts, and the number of entries to num_p.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't
+ * want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdwxacts(int *num_p)
+{
+	List		*all_fdwxacts;
+	ListCell	*lc;
+	FdwXact		fdwxacts;
+	int			num_fdwxacts = 0;
+
+	Assert(num_p != NULL);
+
+	/* Get all entries */
+	all_fdwxacts = get_fdwxacts(InvalidOid, InvalidTransactionId,
+								InvalidOid, InvalidOid, true,
+								true, true);
+
+	if (all_fdwxacts == NIL)
+	{
+		*num_p = 0;
+		return NULL;
+	}
+
+	fdwxacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdwxacts));
+	*num_p = list_length(all_fdwxacts);
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdwxacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdwxacts + num_fdwxacts, fx,
+			   sizeof(FdwXactData));
+		num_fdwxacts++;
+	}
+
+	list_free(all_fdwxacts);
+
+	return fdwxacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respecitively.
+ */
+static List*
+get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			 bool include_indoubt, bool include_in_progress, bool need_lock)
+{
+	int i;
+	List	*fdwxact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact	fdwxact = FdwXactCtl->fdwxacts[i];
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* include in-doubt transaction? */
+		if (!include_indoubt && fdwxact->indoubt)
+			continue;
+
+		/* include in-progress transaction? */
+		if (!include_in_progress && FdwXactIsBeingResolved(fdwxact))
+			continue;
+
+		/* Append it if matched */
+		fdwxact_list = lappend(fdwxact_list, fdwxact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdwxact_list;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	*id;
+	int		id_len = 0;
+
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		/*
+		 * FDW doesn't provide the callback function, generate an unique
+		 * idenetifier.
+		 */
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+			 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifer \"%s\" is too long",
+						id),
+				 errdetail("foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;						/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdwxacts,
+							 serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
+									&read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid =
+		XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid  != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							  fdwxact_data->serverid, fdwxact_data->userid,
+							  fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know
+	 * the xact status right now. Resolver will set it later based on
+	 * the status of local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;	/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdwxact(dbid, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		return;
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(int *newval, void **extra, GucSource source)
+{
+	ForeignTwophaseCommitLevel newForeignTwophaseCommitLevel = *newval;
+
+	/* Parameter check */
+	if (newForeignTwophaseCommitLevel > FOREIGN_TWOPHASE_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	7
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdwxacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdwxacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(PG_PREPARED_FDWXACTS_COLS);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "indoubt",
+						   BOOLOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 7, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdwxacts = get_all_fdwxacts(&num_fdwxacts);
+		status->num_xacts = num_fdwxacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdwxact = &status->fdwxacts[status->cur_xact++];
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdwxact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdwxact->dbid);
+		values[1] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[2] = ObjectIdGetDatum(fdwxact->serverid);
+		values[3] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (fdwxact->status)
+		{
+			case FDWXACT_STATUS_INITIAL:
+				xact_status = "initial";
+				break;
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			case FDWXACT_STATUS_RESOLVED:
+				xact_status = "resolved";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		values[5] = BoolGetDatum(fdwxact->indoubt);
+		values[6] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	FdwXact			fdwxact;
+	FdwXactRslvState	*state;
+	FdwXactStatus		prev_status;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	server = GetForeignServer(serverid);
+	usermapping = GetUserMapping(userid, serverid);
+	state = create_fdwxact_state();
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+	{
+		LWLockRelease(FdwXactLock);
+		PG_RETURN_BOOL(false);
+	}
+
+	state->server = server;
+	state->usermapping = usermapping;
+	state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+
+	SpinLockAcquire(&fdwxact->mutex);
+	prev_status = fdwxact->status;
+	SpinLockRelease(&fdwxact->mutex);
+
+	FdwXactDetermineTransactionFate(fdwxact, false);
+
+	LWLockRelease(FdwXactLock);
+
+	FdwXactResolveForeignTransaction(fdwxact, state, prev_status);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000000..45fb530916
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,644 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to retry resolution.
+ */
+void
+FdwXactLauncherRequestToLaunchForRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			resolver->last_resolved_time = 0;
+			resolver->latch = NULL;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a foreign_xact_resolution_retry_interval
+		 * but always starts when the backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find FdwXact
+		 * entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int unused_slot;
+	int i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (unused_slot > max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	*resolver_dbs;	/* DBs resolver's running on */
+	HTAB	*fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL	ctl;
+	HASH_SEQ_STATUS status;
+	Oid		*entry;
+	bool	launched;
+	int		i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->indoubt)
+			continue;
+
+		hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+		return false;
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign trasaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		 /* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+						WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+						10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Returns activity of all foreign transaction resolvers.
+ */
+Datum
+pg_stat_get_foreign_xact(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000000..9298877f10
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,344 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+bool foreign_xact_resolve_indoubt_xacts;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	MyFdwXactResolver->last_resolved_time = 0;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		PGPROC			*waiter = NULL;
+		TransactionId	waitXid = InvalidTransactionId;
+		TimestampTz		resolutionTs = -1;
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or got the waiter
+		 * that has future resolution time.
+		 */
+		while ((waiter = FdwXactGetWaiter(&resolutionTs, &waitXid)) != NULL)
+		{
+			CHECK_FOR_INTERRUPTS();
+			Assert(TransactionIdIsValid(waitXid));
+
+			if	(resolutionTs > now)
+				break;
+
+			elog(DEBUG2, "resolver got one waiter with xid %u", waitXid);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveTransactionAndReleaseWaiter(MyDatabaseId, waitXid,
+													  waiter);
+			CommitTransactionCommand();
+
+			/* Update my stats */
+			SpinLockAcquire(&(MyFdwXactResolver->mutex));
+			MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+			SpinLockRelease(&(MyFdwXactResolver->mutex));
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz last_resolved_time;
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	last_resolved_time = MyFdwXactResolver->last_resolved_time;
+	timeout = TimestampTzPlusMilliseconds(last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * checked FdwXactWaiterExists.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long	sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index f88d72fd86..982c1a36cc 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -13,6 +13,7 @@ OBJS = \
 	clogdesc.o \
 	committsdesc.o \
 	dbasedesc.o \
+	fdwxactdesc.o \
 	genericdesc.o \
 	gindesc.o \
 	gistdesc.o \
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000000..fe0cef9472
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxactdesc.c
+ *		PostgreSQL global transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdwxactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdwxact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		FdwXactOnDiskData *fdwxact_insert = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_insert->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_insert->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_insert->dbid);
+		appendStringInfo(buf, " local xid: %u,", fdwxact_insert->local_xid);
+		appendStringInfo(buf, " id: %s", fdwxact_insert->fdwxact_id);
+	}
+	else
+	{
+		xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_remove->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_remove->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_remove->dbid);
+		appendStringInfo(buf, " local xid: %u", fdwxact_remove->xid);
+	}
+
+}
+
+const char *
+fdwxact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDWXACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDWXACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 33060f3042..1d4e1c82e1 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 58091f6b52..200cf9d067 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -10,6 +10,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 529976885f..2c9af36bbb 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -850,6 +851,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 	return result;
 }
 
+/*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
 /*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
@@ -2262,6 +2292,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2321,6 +2357,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 5353b6ab0b..5b67056c65 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1218,6 +1219,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1226,6 +1228,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1264,12 +1267,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1427,6 +1431,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2086,6 +2098,10 @@ CommitTransaction(void)
 			break;
 	}
 
+ 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2246,6 +2262,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2333,6 +2350,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2527,6 +2546,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2732,6 +2752,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 6bc1a6b46d..428a974c51 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
@@ -5246,6 +5247,7 @@ BootStrapXLOG(void)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6189,6 +6191,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_wal_senders",
 									 max_wal_senders,
 									 ControlFile->max_wal_senders);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
@@ -6729,14 +6734,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -6928,7 +6934,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7424,6 +7433,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7754,6 +7764,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9029,6 +9042,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9462,8 +9476,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9481,6 +9497,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9497,6 +9514,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9702,6 +9720,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -9901,6 +9920,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index f7800f01a6..b4c1cce1f0 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -332,6 +332,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
@@ -818,6 +821,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_foreign_xact AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_foreign_xact() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 42a147b67d..e3caef7ef9 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2857,8 +2857,14 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
+	}
 
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index 766c9f95c8..43bbe8356d 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1101,6 +1103,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1419,6 +1433,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
+	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
 	/*
 	 * Do the deletion
 	 */
@@ -1572,6 +1595,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index d23f292cb0..690717c34e 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/table.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
@@ -944,7 +945,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 52af1dac5c..3ac56d1678 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,9 +226,31 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
+	}
 
 	return scanstate;
 }
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index cd91f9c8a8..c1ab3d829a 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -549,6 +551,10 @@ ExecInsert(ModifyTableState *mtstate,
 										   NULL,
 										   specToken);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -777,6 +783,10 @@ ldelete:;
 									&tmfd,
 									changingPart);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -1323,6 +1333,10 @@ lreplace:;
 									true /* wait for commit */ ,
 									&tmfd, &lockmode, &update_indexes);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -2382,6 +2396,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index c917ec40ff..0b17505aac 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMappingOid - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,20 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction &&
+		 !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction &&
+		 routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data-wrapper must support both commit and rollback routine or either");
+
+	if (routine->PrepareForeignTransaction &&
+		(!routine->CommitForeignTransaction ||
+		 !routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 5f8a007e73..0a8890a984 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -14,6 +14,8 @@
 
 #include <unistd.h>
 
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -129,6 +131,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index fabcf31de8..0d3932c2cf 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3650,6 +3650,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3853,6 +3859,11 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDWXACT:
+			event_name = "FdwXact";
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -4068,6 +4079,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 9ff2832c00..f92be8387d 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -909,6 +911,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -984,12 +990,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index bc532d027b..6269f384af 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -151,6 +151,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 4829953ee6..6bde7a735a 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(void)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(void)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 13bcbe77de..020eb76b6a 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -93,6 +93,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -248,6 +250,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1312,6 +1315,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1377,6 +1381,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1426,6 +1431,15 @@ GetOldestXmin(Relation rel, int flags)
 		NormalTransactionIdPrecedes(replication_slot_xmin, result))
 		result = replication_slot_xmin;
 
+	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
 	/*
 	 * After locks have been released and vacuum_defer_cleanup_age has been
 	 * applied, check whether we need to back up further to make logical
@@ -3128,6 +3142,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index db47843229..adb276370c 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -49,3 +49,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 CLogTruncationLock					44
+FdwXactLock							45
+FdwXactResolverLock					46
+FdwXactResolutionLock				47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index fff0628e58..af5e418a03 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -421,6 +422,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* Initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -822,6 +827,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 3b85e48333..a0f8498862 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3029,6 +3031,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index ba74bf9f7d..d38c33b64c 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -399,6 +400,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 	{NULL, 0, false}
 };
 
+/*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
 /*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
@@ -725,6 +745,12 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
+	/* FDWXACT_RESOLVER */
+	gettext_noop("Foreign Transaction Management / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2370,6 +2396,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FDWXACT_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FDWXACT_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4413,6 +4485,16 @@ static struct config_enum ConfigureNamesEnum[] =
 		NULL, assign_synchronous_commit, NULL
 	},
 
+	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
 	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 9541879c1f..22e014aecd 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -341,6 +343,20 @@
 #max_sync_workers_per_subscription = 2	# taken from max_logical_replication_workers
 
 
+#------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
 #------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index f08a49c9dd..dd8878025b 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 1f6d8939be..49dc5a519f 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -210,6 +210,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 19e21ab491..9ae3bfe4dd 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -301,6 +301,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 2e286f6339..c5ee22132e 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -710,6 +710,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -914,6 +915,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/fdwxactdesc.c b/src/bin/pg_waldump/fdwxactdesc.c
new file mode 120000
index 0000000000..ce8c21880c
--- /dev/null
+++ b/src/bin/pg_waldump/fdwxactdesc.c
@@ -0,0 +1 @@
+../../../src/backend/access/rmgrdesc/fdwxactdesc.c
\ No newline at end of file
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca4b1..b616cea347 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000000..147d41c708
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,165 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL global transaction manager
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDWXACT_H
+#define FDWXACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* fdwXactState */
+#define	FDWXACT_NOT_WAITING		0
+#define	FDWXACT_WAITING			1
+#define	FDWXACT_WAIT_COMPLETE	2
+
+/* Flag passed to FDW transaction management APIs */
+#define FDWXACT_FLAG_ONEPHASE		0x01	/* transaction can commit/rollback
+											   without preparation */
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER,		/* use twophase commit where available */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support
+										   twophase commit */
+} ForeignTwophaseCommitLevel;
+
+/* Enum to track the status of foreign transaction */
+typedef enum
+{
+	FDWXACT_STATUS_INVALID,
+	FDWXACT_STATUS_INITIAL,
+	FDWXACT_STATUS_PREPARING,		/* foreign transaction is being prepared */
+	FDWXACT_STATUS_PREPARED,		/* foreign transaction is prepared */
+	FDWXACT_STATUS_COMMITTING,		/* foreign prepared transaction is to
+									 * be committed */
+	FDWXACT_STATUS_ABORTING,		/* foreign prepared transaction is to be
+									 * aborted */
+	FDWXACT_STATUS_RESOLVED
+} FdwXactStatus;
+
+typedef struct FdwXactData *FdwXact;
+
+/*
+ * Shared memory state of a single foreign transaction.
+ */
+typedef struct FdwXactData
+{
+	FdwXact			fdwxact_free_next;	/* Next free FdwXact entry */
+
+	Oid				dbid;			/* database oid where to find foreign server
+									 * and user mapping */
+	TransactionId	local_xid;		/* XID of local transaction */
+	Oid				serverid;		/* foreign server where transaction takes
+									 * place */
+	Oid				userid;			/* user who initiated the foreign
+									 * transaction */
+	Oid				umid;
+	bool			indoubt;		/* Is an in-doubt transaction? */
+	slock_t			mutex;			/* Protect the above fields */
+
+	/* The status of the foreign transaction, protected by FdwXactLock */
+	FdwXactStatus 	status;
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;		/* XLOG offset of inserting this entry start */
+	XLogRecPtr	insert_end_lsn;		/* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN];		/* prepared transaction identifier */
+} FdwXactData;
+
+/*
+ * Shared memory layout for maintaining foreign prepared transaction entries.
+ * Adding or removing FdwXact entry needs to hold FdwXactLock in exclusive mode,
+ * and iterating fdwXacts needs that in shared mode.
+ */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		free_fdwxacts;
+
+	/* Number of valid foreign transaction entries */
+	int			num_fdwxacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdwxacts[FLEXIBLE_ARRAY_MEMBER];		/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* State data for foreign transaction resolution, passed to FDW callbacks */
+typedef struct FdwXactRslvState
+{
+	/* Foreign transaction information */
+	char	*fdwxact_id;
+
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	int		flags;			/* OR of FDWXACT_FLAG_xx flags */
+} FdwXactRslvState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+/* Function declarations */
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern bool fdwxact_exists(Oid dboid, Oid serverid, Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern void PreCommit_FdwXacts(void);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+extern void FdwXactResolveTransactionAndReleaseWaiter(Oid dbid, TransactionId xid,
+													  PGPROC *waiter);
+extern bool FdwXactResolveInDoubtTransactions(Oid dbid);
+extern PGPROC *FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p);
+extern void FdwXactCleanupAtProcExit(void);
+extern void RegisterFdwXactByRelId(Oid relid, bool modified);
+extern void RegisterFdwXactByServerId(Oid serverid, bool modified);
+extern void FdwXactMarkForeignServerAccessed(Oid relid, bool modified);
+extern bool check_foreign_twophase_commit(int *newval, void **extra,
+										  GucSource source);
+extern bool FdwXactWaiterExists(Oid dbid);
+
+#endif   /* FDWXACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000000..dd0f5d16ff
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,29 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherRequestToLaunch(void);
+extern void FdwXactLauncherRequestToLaunchForRetry(void);
+extern void FdwXactLaunchOrWakeupResolver(void);
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+
+
+#endif	/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000000..2607654024
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int foreign_xact_resolver_timeout;
+
+#endif		/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000000..39ca66beef
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,54 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDWXACT_INSERT	0x00
+#define XLOG_FDWXACT_REMOVE	0x10
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDWXACT_ID_MAX_LEN 200
+
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdwxact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+	bool		force;
+} xl_fdwxact_remove;
+
+extern void fdwxact_redo(XLogReaderState *record);
+extern void fdwxact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdwxact_identify(uint8 info);
+
+#endif	/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000000..55fc970b69
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,66 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/resovler_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t	pid;	/* this resolver's PID, or 0 if not active */
+	Oid		dbid;	/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool	in_use;
+
+	/* Stats */
+	TimestampTz	last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t	mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	*latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/* Foreign transaction resolution queue. Protected by FdwXactLock */
+	SHM_QUEUE	fdwxact_queue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch		*launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+#define SizeOfFdwXactRslvCtlData \
+	(offsetof(FdwXactRslvCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif	/* RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 3c0db2ccf5..5798b4cd99 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Foreign Transactions", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 02b5315c43..e8c094d708 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index cb5c4935d2..a75e6998f0 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -108,6 +108,13 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
+/*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index e295dc65fb..d1ce20242f 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -232,6 +232,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index cf7d4485e9..f2174a0208 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index ac8f64b219..1072c38aa6 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5184,6 +5184,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '9705', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_foreign_xact', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_foreign_xact' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5897,6 +5904,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '9706', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o,o}',
+  proargnames => '{dbid,xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '9707', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '9708', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -6015,6 +6040,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '9709',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 822686033e..c7b33d72ec 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 4de157c19c..91c2276915 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -69,6 +69,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 											   bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 														 bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index fe076d823d..d82d8f7abc 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -776,6 +776,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -853,7 +855,9 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDWXACT,
+	WAIT_EVENT_FDWXACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -933,6 +937,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index 281e1db725..c802201193 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -16,6 +16,7 @@
 
 #include "access/clog.h"
 #include "access/xlogdefs.h"
+#include "datatype/timestamp.h"
 #include "lib/ilist.h"
 #include "storage/latch.h"
 #include "storage/lock.h"
@@ -152,6 +153,16 @@ struct PGPROC
 	int			syncRepState;	/* wait state for sync rep */
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
+	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
 	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index 8f67b860e7..deb293c1a9 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -125,4 +127,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index d68976fafa..d5fec50969 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,9 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
+	FDWXACT_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index c9cc569404..ed229d5a67 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1341,6 +1341,14 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.dbid,
+    f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(dbid, xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
@@ -1841,6 +1849,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_foreign_xact| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_foreign_xact() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_gssapi| SELECT s.pid,
     s.gss_auth AS gss_authenticated,
     s.gss_princ AS principal,
-- 
2.23.0

From 3363abd531595233fb59e0ab6078a011ab8060e9 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 5 Dec 2019 17:01:08 +0900
Subject: [PATCH v26 3/5] Documentation update.

Original Author: Masahiko Sawada <sawada.mshk@gmail.com>
---
 doc/src/sgml/catalogs.sgml                | 145 +++++++++++++
 doc/src/sgml/config.sgml                  | 146 ++++++++++++-
 doc/src/sgml/distributed-transaction.sgml | 158 +++++++++++++++
 doc/src/sgml/fdwhandler.sgml              | 236 ++++++++++++++++++++++
 doc/src/sgml/filelist.sgml                |   1 +
 doc/src/sgml/func.sgml                    |  89 ++++++++
 doc/src/sgml/monitoring.sgml              |  60 ++++++
 doc/src/sgml/postgres.sgml                |   1 +
 doc/src/sgml/storage.sgml                 |   6 +
 9 files changed, 841 insertions(+), 1 deletion(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 55694c4368..1b720da03d 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8267,6 +8267,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
       <entry>open cursors</entry>
      </row>
 
+     <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
      <row>
       <entry><link linkend="view-pg-file-settings"><structname>pg_file_settings</structname></link></entry>
       <entry>summary of configuration file contents</entry>
@@ -9712,6 +9717,146 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with that this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on that the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>initial</literal> : Initial status.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>resolved</literal> : This foreign transaction has been resolved.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-dbout status and
+       needs to be resolved by calling <function>pg_resolve_fdwxact</function>
+       function.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 53ac14490a..69778750f3 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4378,7 +4378,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -8818,6 +8817,151 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign
+         transaction to be resolved before the command returns a "success"
+         indication to the client. Valid values are <literal>required</literal>,
+         <literal>prefer</literal> and <literal>disabled</literal>. The default
+         setting is <literal>disabled</literal>. Setting to
+         <literal>disabled</literal> don't use two-phase commit protocol to
+         commit or rollback distributed transactions. When set to
+         <literal>required</literal> the distributed transaction strictly
+         requires that all written servers can use two-phase commit protocol.
+         That is, the distributed transaction cannot commit if even one server
+         does not support the transaction management callback routines
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. Note that when <literal>disabled</literal> or
+         <literal>prefer</literal> there can be risk of database consistency
+         among all servers that involved in the distributed transaction when some
+         foreign server crashes during committing the distributed transaction.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero
+         value to set this parameter either <literal>required</literal> or
+         <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000000..350b1afe68
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,158 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).
+    A <productname>PostgreSQL</productname> server that received SQL is called
+    <firstterm>coordinator node</firstterm> who is responsible for coordinating
+    all the partipanting transactions. Using two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed
+    transaction manager prepares all transaction on the foreign servers if
+    two-phase commit is required. Two-phase commit is required when the
+    transaction modifies data on two or more servers including the local server
+    itself and <xref linkend="guc-foreign-twophase-commit"/>is
+    <literal>required</literal> or <literal>prefer</literal>. If all preparations
+    on foreign servers got successful go to the next step. Any failure happens
+    in this step <productname>PostgreSQL</productname> changes to rollback, then
+    rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the
+    transaction locally. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes rollback, then rollback all
+    transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolution">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that is
+    responsible for foreign transaction resolution. They commit or rollback all
+    prepared transaction on foreign servers if the coordinator received agreement
+    messages from all foreign servers during the first step.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on one database of the coordinator side. On failure during resolution, they
+    retries to resolve at an interval of
+    <varname>foreign_transaction_resolution_interval</varname> time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped. So to drop the database, you can call
+     <function>pg_stop_foreign_xact_resovler</function> function before dropping
+     the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>Manual Resolution of In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, distributed transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the coordinator node crashed during either preparing or
+    resolving distributed transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="pg-stat-foreign-xact-view"/>
+    view. These foreign transactions need to be resolved by using
+    <function>pg_resolve_foriegn_xact</function> function.
+    <productname>PostgreSQL</productname> doesn't have facilities to automatially
+    resolve in-doubt transactions. These behavior might change in a future release.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-monitoring">
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-foreign-xact-view"><literal>pg_stat_foreign_xact</literal></link>
+    view. This view contains one row for every foreign transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 6587678af2..dd0358ef22 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1415,6 +1415,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1894,4 +2015,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 3da2365ea9..80a87fa5d1 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 57a1539506..b9a918b9ee 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -22355,6 +22355,95 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index a3c5f86b7e..65938e81ca 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -368,6 +368,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_foreign_xact</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1236,6 +1244,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry><literal>CheckpointerMain</literal></entry>
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
+        <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
         <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
@@ -1459,6 +1479,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry><literal>SyncRep</literal></entry>
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
+        <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
         <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
@@ -2359,6 +2383,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-view" xreflabel="pg_stat_foreign_xact">
+   <title><structname>pg_stat_foreign_xact</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index e59cba7997..dee3f72f7e 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -163,6 +163,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 1c19e863d2..3f4c806ed1 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -83,6 +83,12 @@ Item
   subsystem</entry>
 </row>
 
+<row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
 <row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
-- 
2.23.0

From 84f81fdcb2bd823e34edba79c81c29871d7906fb Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 5 Dec 2019 17:01:15 +0900
Subject: [PATCH v26 4/5] postgres_fdw supports atomic commit APIs.

Original Author: Masahiko Sawada <sawada.mshk@gmail.com>
---
 contrib/postgres_fdw/Makefile                 |   7 +-
 contrib/postgres_fdw/connection.c             | 604 +++++++++++-------
 .../postgres_fdw/expected/postgres_fdw.out    | 265 +++++++-
 contrib/postgres_fdw/fdwxact.conf             |   3 +
 contrib/postgres_fdw/postgres_fdw.c           |  21 +-
 contrib/postgres_fdw/postgres_fdw.h           |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql     | 120 +++-
 doc/src/sgml/postgres-fdw.sgml                |  45 ++
 8 files changed, 822 insertions(+), 250 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index ee8a80a392..91fa6e39fc 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -16,7 +16,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -29,3 +29,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index 27b86a03f8..0b07e6c5cc 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2019, PostgreSQL Global Development Group
  *
@@ -12,6 +12,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "catalog/pg_user_mapping.h"
@@ -54,6 +55,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -67,17 +69,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 								   SubTransactionId mySubid,
 								   SubTransactionId parentSubid,
@@ -89,24 +87,26 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 									 bool ignore_errors);
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 									 PGresult **result);
-
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -126,7 +126,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -134,12 +133,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -153,6 +146,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -180,6 +188,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -188,6 +197,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -198,6 +208,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -205,11 +224,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -412,7 +459,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -639,193 +686,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 	PG_END_TRY();
 }
 
-/*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow any remote transactions, since it's not
-					 * very reasonable to hold them open until the prepared
-					 * transaction is committed.  For the moment, throw error
-					 * unconditionally; later we might allow read-only cases.
-					 * Note that the error will cause us to come right back
-					 * here with event == XACT_EVENT_ABORT, so we'll clean up
-					 * the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot PREPARE a transaction that has operated on postgres_fdw foreign tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
 /*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
@@ -842,10 +702,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -856,6 +712,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1190,3 +1050,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recusively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 48282ab151..0ee91a49ac 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -179,15 +198,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8781,16 +8802,226 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 
 -- Clean-up
 RESET enable_partitionwise_aggregate;
--- Two-phase transactions are not supported.
+
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
 BEGIN;
-SELECT count(*) FROM ft1;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
  count 
 -------
-   822
+     0
 (1 row)
 
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
-ERROR:  cannot PREPARE a transaction that has operated on postgres_fdw foreign tables
-ROLLBACK;
-WARNING:  there is no transaction in progress
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000000..3fdbf93cdb
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index bdc21b36d1..9c63f0aa3b 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include <limits.h>
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -504,7 +505,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 							  const PgFdwRelationInfo *fpinfo_o,
 							  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -558,6 +558,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1434,7 +1439,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2372,7 +2377,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2746,7 +2751,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3566,7 +3571,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4441,7 +4446,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4527,7 +4532,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4755,7 +4760,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index ea052872c3..d7ba45c8d2 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "libpq-fe.h"
@@ -129,7 +130,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -137,6 +138,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 							   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int	ExtractConnectionOptions(List *defelems,
@@ -203,6 +207,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 									bool is_subquery,
 									List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 1c5c37b783..572077c57c 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2480,9 +2503,98 @@ SELECT b, avg(a), max(a), count(*) FROM pagg_tab GROUP BY b HAVING sum(a) < 700
 -- Clean-up
 RESET enable_partitionwise_aggregate;
 
--- Two-phase transactions are not supported.
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
 BEGIN;
-SELECT count(*) FROM ft1;
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
+INSERT INTO ft7_2pc VALUES(1);
 ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 1d4bafd9f0..362f7be9e3 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -441,6 +441,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -468,6 +505,14 @@
    managed by creating corresponding remote savepoints.
   </para>
 
+  <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
   <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
-- 
2.23.0

From 639d9156323594430ec4b2217a95bfcf08195e9d Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 5 Dec 2019 17:01:26 +0900
Subject: [PATCH v26 5/5] Add regression tests for atomic commit.

Original Author: Masahiko Sawada <sawada.mshk@gmail.com>
---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index e66e69521f..b17429f501 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000000..9af9bb81dc
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 297b8fbd6f..82a1e7d541 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2336,9 +2336,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2353,7 +2356,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.23.0

From 42eea5f76a41e582e66ac532645fd95faa244bb2 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 5 Dec 2019 17:01:26 +0900
Subject: [PATCH v27 5/5] Add regression tests for atomic commit.

Original Author: Masahiko Sawada <sawada.mshk@gmail.com>
---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index fa8e031526..d47d96975b 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000000..9af9bb81dc
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 92bd28dc5a..de8a292bba 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2336,9 +2336,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2353,7 +2356,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.23.0

From df8e4b75e049d274188ad412dfd419d23c0d881a Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 5 Dec 2019 16:59:47 +0900
Subject: [PATCH v27 1/5] Keep track of writing on non-temporary relation

Original Author: Masahiko Sawada <sawada.mshk@gmail.com>
---
 src/backend/executor/nodeModifyTable.c | 12 ++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 18 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 59d1a31c97..c0a15c3412 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -587,6 +587,10 @@ ExecInsert(ModifyTableState *mtstate,
 							   estate->es_output_cid,
 							   0, NULL);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -938,6 +942,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1447,6 +1455,10 @@ lreplace:;
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL, NIL);
 	}
 
+	/* Make note that we've wrote on non-temprary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	if (canSetTag)
 		(estate->es_processed)++;
 
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7ee04babc2..a04fc70326 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -102,6 +102,12 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
-- 
2.23.0

From 1c592918acd2da8dab0f5ffba167a70fcc39be5d Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 5 Dec 2019 17:00:50 +0900
Subject: [PATCH v27 2/5] Support atomic commit among multiple foreign servers.

Original Author: Masahiko Sawada <sawada.mshk@gmail.com>
---
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  130 +
 src/backend/access/fdwxact/fdwxact.c          | 2815 +++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  644 ++++
 src/backend/access/fdwxact/resolver.c         |  343 ++
 src/backend/access/rmgrdesc/Makefile          |    1 +
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   58 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/copy.c                   |    6 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    8 +
 src/backend/executor/nodeForeignscan.c        |   24 +
 src/backend/executor/nodeModifyTable.c        |   18 +
 src/backend/foreign/foreign.c                 |   57 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   20 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   82 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/fdwxactdesc.c              |    1 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  164 +
 src/include/access/fdwxact_launcher.h         |   29 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   54 +
 src/include/access/resolver_internal.h        |   66 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   29 +
 src/include/foreign/fdwapi.h                  |   12 +
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    9 +-
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    3 +
 src/test/regress/expected/rules.out           |   13 +
 55 files changed, 4913 insertions(+), 18 deletions(-)
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100644 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 120000 src/bin/pg_waldump/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a8bb..49480dd039 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000000..0207a66fb4
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000000..46ccb7eeae
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,130 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a conssitent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+--------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consisnts
+of the following four steps:
+
+1. Foriegn Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+we record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE all foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared tranascation on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step(commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itselft to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rolbacked the resolver process wake up the waiter.
+
+
+API Contract With Transaction Management Callback Functions
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForiegnTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for either PREPARE, COMMIT or
+ROLLBACK the trasaction on the foreign server respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transactio in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions has an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_INITIAL
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_PREPARING, FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING
+before the foreign transaction is prepared, committed and aborted by FDW
+callback functions respectively(*1). And the status then changes to
+FDWXACT_STATUS_RESOLVED once the foreign transaction are resolved, and then
+the corresponding FdwXact entry is removed with WAL logging. If failed during
+processing foreign transaction (i.g. preparing, committing or aborting) the
+status changes back to the previous status. Therefore the status
+FDWXACT_STATUS_xxxING appear only during the foreign transaction is being
+processed by an FDW callback function.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*2). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                      INVALID                       |
+ +----------------------------------------------------+
+    |                      |                       |
+    |                      v                       |
+    |           +---------------------+            |
+    |           |       INITIAL       |            |
+    |           +---------------------+            |
+   (*2)                    |                      (*2)
+    |                      v                       |
+    |           +---------------------+            |
+    |           |    PREPARING(*1)    |            |
+    |           +---------------------+            |
+    |                      |                       |
+    v                      v                       v
+ +----------------------------------------------------+
+ |                      PREPARED                      |
+ +----------------------------------------------------+
+           |                               |
+           v                               v
+ +--------------------+          +--------------------+
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |
+ +--------------------+          +--------------------+
+           |                               |
+           v                               v
+ +----------------------------------------------------+
+ |                      RESOLVED                      |
+ +----------------------------------------------------+
+
+(*1) Status that appear only during being processed by FDW
+(*2) Paths for recovered FdwXact entries
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100644
index 0000000000..66ba4fe953
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2815 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers automically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit. The foreign servers are
+ * registered if FDW has both CommitForeignTransaciton API and
+ * RollbackForeignTransactionAPI. Registered participant servers are identified
+ * by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request. We have one queue
+ * of which elements are ordered by the timestamp that they expect to be
+ * processed at. Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp that they expects to be processed.
+ * Similary if failed to resolve them, it enqueues again with new timestamp
+ * (its timestamp + foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction). Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by pg_resovle_fdwxact()
+ * function.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or
+ * more servers including itself. In other case, all foreign transactions are
+ * committed or rolled back during pre-commit.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed by FDW, the corresponding
+ * FdwXact entry is update. In order to protect the entry from concurrent
+ * removing we need to hold a lock on the entry or a lock for entire global
+ * array. However, we don't want to hold the lock during FDW is processing the
+ * foreign transaction that may take a unpredictable time. To avoid this, the
+ * in-memory data of foreign transaction follows a locking model based on
+ * four linked concepts:
+ *
+ * * A foreign transaction's status variable is switched using the LWLock
+ *   FdwXactLock, which need to be hold in exclusive mode when updating the
+ *   status, while readers need to hold it in shared mode when looking at the
+ *   status.
+ * * A process who is going to update FdwXact entry cannot process foreign
+ *   transaction that is being resolved.
+ * * So setting the status to FDWACT_STATUS_PREPARING,
+ *   FDWXACT_STATUS_COMMITTING or FDWXACT_STATUS_ABORTING, which makes foreign
+ *   transaction in-progress states, means to own the FdwXact entry, which
+ *   protect it from updating/removing by concurrent writers.
+ * * Individual fields are protected by mutex where only the backend owning
+ *   the foreign transaction is authorized to update the fields from its own
+ *   one.
+
+ * Therefore, before doing PREPARE, COMMIT PREPARED or ROLLBACK PREPARED a
+ * process who is going to call transaction callback functions needs to change
+ * the status to the corresponding status above while holding FdwXactLock in
+ * exclusive mode, and call callback function after releasing the lock.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk. FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon. We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts. If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to dink.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Atomic commit is enabled by configuration */
+#define IsForeignTwophaseCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	(IsForeignTwophaseCommitEnabled() && \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED))
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define IsSeverCapableOfTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/* Check the FdwXact is begin resolved */
+#define FdwXactIsBeingResolved(fx) \
+	(((((FdwXact)(fx))->status) == FDWXACT_STATUS_PREPARING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_COMMITTING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_ABORTING))
+
+/*
+ * Structure to bundle the foreign transaction participant. This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry
+	 * is not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char			*fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool			modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function	prepare_foreign_xact_fn;
+	CommitForeignTransaction_function	commit_foreign_xact_fn;
+	RollbackForeignTransaction_function	rollback_foreign_xact_fn;
+	GetPrepareId_function				get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit. This list
+ * has only foreign servers that provides transaction management callbacks,
+ * that is CommitForeignTransaction and RollbackForeignTransaction.
+ */
+static List *FdwXactParticipants = NIL;
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int	max_prepared_foreign_xacts = 0;
+int	max_foreign_xact_resolvers = 0;
+int foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid,
+										 FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+												 bool for_commit);
+static void FdwXactResolveForeignTransaction(FdwXact fdwxact,
+											 FdwXactRslvState *state,
+											 FdwXactStatus fallback_status);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid,	void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static void FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock);
+static bool is_foreign_twophase_commit_required(void);
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static List *get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						  bool including_indoubts, bool include_in_progress,
+						  bool need_lock);
+static FdwXact get_all_fdwxacts(int *num_p);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXact get_fdwxact_to_resolve(Oid dbid, TransactionId xid);
+static FdwXactRslvState *create_fdwxact_state(void);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation		rel;
+	Oid				serverid;
+	Oid				userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign transaction identified by given server id and user id.
+ * Registered foreign transactions are managed by the global transaction
+ * manager until the end of the transaction.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant	*fdw_part;
+	ForeignServer 		*foreign_server;
+	UserMapping			*user_mapping;
+	MemoryContext		old_ctx;
+	FdwRoutine			*routine;
+	ListCell	   		*lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Don't register foreign server if it doesn't provide both commit and
+	 * rollback transaction management callbacks.
+	 */
+	if (!routine->CommitForeignTransaction ||
+		!routine->RollbackForeignTransaction)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		pfree(routine);
+		return;
+	}
+
+	/*
+	 * Remember we touched the foreign server that is not capable of two-phase
+	 * commit.
+	 */
+	if (!routine->PrepareForeignTransaction)
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+	foreign_server = GetForeignServer(serverid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&(fdwxacts[cnt].mutex));
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * If foreign twophase commit is required, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign server's FDWs to support two-phase commit protocol and ask them to
+ *  prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and for 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_twophase_commit;
+	ListCell	*lc = NULL;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * we require all modified server have to be capable of two-phase
+	 * commit protocol.
+	 */
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false
+	 * if foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = is_foreign_twophase_commit_required();
+
+	/*
+	 * Firstly, we consider to commit foreign transactions in one-phase.
+	 */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool	commit = false;
+
+		/* Can commit in one-phase if two-phase commit is not requried */
+		if (!need_twophase_commit)
+			commit = true;
+
+		/* Non-modified foreign transaction always can be committed in one-phase */
+		if (!fdw_part->modified)
+			commit = true;
+
+		/*
+		 * In 'prefer' case, non-twophase-commit capable server can be
+		 * committed in one-phase.
+		 */
+		if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER &&
+			!IsSeverCapableOfTwophaseCommit(fdw_part))
+			commit = true;
+
+		if (commit)
+		{
+			/* Commit the foreign transaction in one-phase */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, true);
+
+			/* Delete it from the participant list */
+			FdwXactParticipants = foreach_delete_current(FdwXactParticipants,
+														 lc);
+			continue;
+		}
+	}
+
+	/* All done if we committed all foreign transactions */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * Secondary, if only one transaction is remained in the participant list
+	 * and we didn't modified the local data we can commit it without
+	 * preparation.
+	 */
+	if (list_length(FdwXactParticipants) == 1 &&
+		(MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) == 0)
+	{
+		/* Commit the foreign transaction in one-phase */
+		FdwXactOnePhaseEndForeignTransaction(linitial(FdwXactParticipants),
+											 true);
+
+		/* All foreign transaction must be committed */
+		list_free(FdwXactParticipants);
+		return;
+	}
+
+	/*
+	 * Finally, prepare foreign transactions. Note that we keep
+	 * FdwXactParticipants until the end of transaction.
+	 */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here. If any error occurs, we rollback
+ * non-prepared foreign trasactions and leave others to the resolver.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell		*lcell;
+	TransactionId	xid;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepread foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	xid = GetTopTransactionId();
+
+	/* Loop over the foreign connections */
+	foreach(lcell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lcell);
+		FdwXactRslvState 	*state;
+		FdwXact		fdwxact;
+
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, xid);
+
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the FDWXACT_STATUS_PREPARING
+		 * status. Registration persists this information to the disk and logs
+		 * (that way relaying it on standby). Thus in case we loose connectivity
+		 * to the foreign server or crash ourselves, we will remember that we
+		 * might have prepared transaction on the foreign server and try to
+		 * resolve it when connectivity is restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before persisting
+		 * the information to the disk and crash in-between these two steps,
+		 * we will forget that we prepared the transaction on the foreign server
+		 * and will not be able to resolve it after the crash. Hence persist
+		 * first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(xid, fdw_part);
+
+		state = create_fdwxact_state();
+		state->server = fdw_part->server;
+		state->usermapping = fdw_part->usermapping;
+		state->fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+
+		/* Update the status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		Assert(fdwxact->status == FDWXACT_STATUS_INITIAL);
+		fdwxact->status = FDWXACT_STATUS_PREPARING;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 *
+		 * During abort processing, we might try to resolve a never-preapred
+		 * transaction, and get an error. This is fine as long as the FDW
+		 * provides us unique prepared transaction identifiers.
+		 */
+		PG_TRY();
+		{
+			fdw_part->prepare_foreign_xact_fn(state);
+		}
+		PG_CATCH();
+		{
+			/* failed, back to the initial state */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			fdwxact->status = FDWXACT_STATUS_INITIAL;
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		/* succeeded, update status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * One-phase commit or rollback the given foreign transaction participant.
+ */
+static void
+FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+									 bool for_commit)
+{
+	FdwXactRslvState *state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state = create_fdwxact_state();
+	state->server = fdw_part->server;
+	state->usermapping = fdw_part->usermapping;
+	state->flags = FDWXACT_FLAG_ONEPHASE;
+
+	/*
+	 * Commit or rollback foreign transaction in one-phase. Since we didn't
+	 * insert FdwXact entry for this transaction we don't need to care
+	 * failures. On failure we change to rollback.
+	 */
+	if (for_commit)
+		fdw_part->commit_foreign_xact_fn(state);
+	else
+		fdw_part->rollback_foreign_xact_fn(state);
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact				fdwxact;
+	FdwXactOnDiskData	*fdwxact_file_data;
+	MemoryContext		old_context;
+	int					data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							fdw_part->usermapping->userid,
+							fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_INITIAL;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				Oid umid, char *fdwxact_id)
+{
+	int i;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("duplicate entry with transaction id %u, serverid %u, userid %u",
+								   xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->dbid = dbid;
+	fdwxact->local_xid = xid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	fdwxact->indoubt = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (FdwXactIsBeingResolved(fdwxact))
+		elog(ERROR, "cannot remove fdwxact entry that is beging resolved");
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("failed to find entry for xid %u, foreign server %u, and user %u",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the
+		 * WAL record is inserted could complete without fsync'ing our
+		 * state file.  (This is essentially the same kind of race condition
+		 * as the COMMIT-to-clog-write case that RecordTransactionCommit
+		 * uses delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and
+		 * remove the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/*
+		 * Now we can mark ourselves as out of the commit critical section: a
+		 * checkpoint starting after this will certainly see the gxact as a
+		 * candidate for fsyncing.
+		 */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true and set FdwXactAtomicCommitReady to true if the current transaction
+ * modified data on two or more servers in FdwXactParticipants and
+ * local server itself.
+ */
+static bool
+is_foreign_twophase_commit_required(void)
+{
+	ListCell*	lc;
+	int			nserverswritten = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		++nserverswritten;
+
+	/*
+	 * Atomic commit is required if we modified data on two or more
+	 * participants.
+	 */
+	if (nserverswritten <= 1)
+		return false;
+
+	ForeignTwophaseCommitIsRequired = true;
+	return true;
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int	i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell *cell;
+	int		n_lefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant	*fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		if (!fdw_part->fdwxact)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in FdwXactParticipants
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget. So we need to check if the entries are still
+		 * associated with the transaction.
+		 */
+		SpinLockAcquire(&fdwxact->mutex);
+		if (fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+			fdwxact->indoubt = true;
+			n_lefts++;
+		}
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (n_lefts > 0)
+	{
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", n_lefts);
+		FdwXactComputeRequiredXmin();
+	}
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * Wait for the foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char		*new_status = NULL;
+	const char	*old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsForeignTwophaseCommitRequested())
+		return;
+
+	/*
+	 * Also exit if the transaction itself has no foreign transaction
+	 * participants.
+	 */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction
+	 * resolution.
+	 */
+	if (update_process_title)
+	{
+		int len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0';	/* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The latter
+		 * would lead the client to believe that the distributed transaction
+		 * aborted, which is not true: it's already committed locally. The
+		 * former is no good either: the client has requested committing a
+		 * distributed transaction, and is entitled to assume that a acknowledged
+		 * commit is also commit on all foreign servers, which might not be
+		 * true. So in this case we issue a WARNING (which some clients may
+		 * be able to interpret) and shut off further output. We do NOT reset
+		 * PorcDiePending, so that the process will die after the commit is
+		 * cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve them
+		 * later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lcell;
+
+	if (!is_commit)
+	{
+		foreach (lcell, FdwXactParticipants)
+		{
+			FdwXactParticipant	*fdw_part = lfirst(lcell);
+
+			/*
+			 * If the foreign transaction has FdwXact entry we might have
+			 * prepared it. Skip already-prepared foreign transaction because
+			 * it has closed its transaction. But we are not sure that foreign
+			 * transaction with status == FDWXACT_STATUS_PREPARING has been
+			 * prepared or not. So we call the rollback API to close its
+			 * transaction for safety. The prepared foreign transaction that
+			 * we might have will be resolved by the foreign transaction
+			 * resolver.
+			 */
+			if (fdw_part->fdwxact)
+			{
+				bool is_prepared;
+
+				LWLockAcquire(FdwXactLock, LW_SHARED);
+				is_prepared = fdw_part->fdwxact &&
+					fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED;
+				LWLockRelease(FdwXactLock);
+
+				if (is_prepared)
+					continue;
+			}
+
+			/* One-phase rollback foreign transaction */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, false);
+		}
+	}
+
+	/*
+	 * In commit cases, we have already prepared foreign transactions during
+	 * pre-commit phase. And these prepared transactions will be resolved by
+	 * the resolver process.
+	 */
+
+	ForgetAllFdwXactParticipants();
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Prepare foreign transactions.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (is_foreign_twophase_commit_required() &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p)
+{
+	PGPROC *proc;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+			break;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+	{
+		*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+		*waitXid_p = proc->fdwXactWaitXid;
+	}
+	else
+	{
+		*nextResolutionTs_p = -1;
+		*waitXid_p = InvalidTransactionId;
+	}
+
+	LWLockRelease(FdwXactResolutionLock);
+
+	return proc;
+}
+
+/*
+ * Get one FdwXact entry to resolve. This function intended to be used when
+ * a resolver process get FdwXact entries to resolve. So we search entries
+ * while not including in-doubt transactions and in-progress transactions.
+ */
+static FdwXact
+get_fdwxact_to_resolve(Oid dbid, TransactionId xid)
+{
+	List *fdwxacts = NIL;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Don't include both in-doubt transactions and in-progress transactions */
+	fdwxacts = get_fdwxacts(dbid, xid, InvalidOid, InvalidOid,
+							false, false, false);
+
+	return fdwxacts == NIL ? NULL : (FdwXact) linitial(fdwxacts);
+}
+
+/*
+ * Resolve one distributed transaction on the given database . The target
+ * distributed transaction is fetched from the waiting queue and its transaction
+ * participants are fetched from the global array.
+ *
+ * Release the waiter and return true after we resolved the all of the foreign
+ * transaction participants. On failure, we re-enqueue the waiting backend after
+ * incremented the next resolution time.
+ */
+void
+FdwXactResolveTransaction(Oid dbid, TransactionId xid, PGPROC *waiter)
+{
+	FdwXact	fdwxact;
+
+	Assert(TransactionIdIsValid(xid));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	while ((fdwxact = get_fdwxact_to_resolve(MyDatabaseId, xid)) != NULL)
+	{
+		FdwXactRslvState *state;
+		ForeignServer *server;
+		UserMapping	*usermapping;
+
+		CHECK_FOR_INTERRUPTS();
+
+		server = GetForeignServer(fdwxact->serverid);
+		usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+
+		state = create_fdwxact_state();
+		SpinLockAcquire(&fdwxact->mutex);
+		state->server = server;
+		state->usermapping = usermapping;
+		state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+		SpinLockRelease(&fdwxact->mutex);
+
+		FdwXactDetermineTransactionFate(fdwxact, false);
+
+		/* Do not hold during foreign transaction resolution */
+		LWLockRelease(FdwXactLock);
+
+		PG_TRY();
+		{
+			/*
+			 * Resolve the foreign transaction. When committing or aborting
+			 * prepared foreign transactions the previous status is always
+			 * FDWXACT_STATUS_PREPARED.
+			 */
+			FdwXactResolveForeignTransaction(fdwxact, state,
+											 FDWXACT_STATUS_PREPARED);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDWXACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				waiter->fdwXactNextResolutionTs =
+					TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+												foreign_xact_resolution_retry_interval);
+				FdwXactQueueInsert(waiter);
+			}
+			LWLockRelease(FdwXactResolutionLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved one foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter
+	 * could already be detached if user cancelled to wait before
+	 * resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId	wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/* Wake up the waiter only when we have set state and removed from queue */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Determine whether the given foreign transaction should be committed or
+ * rolled back according to the result of the local transaction. This function
+ * changes fdwxact->status so the caller must hold FdwXactLock in exclusive
+ * mode or passing need_lock with true.
+ */
+static void
+FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock)
+{
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * The being resolved transaction must be either that has been cancelled
+	 *  and marked as in-doubt or that has been prepared.
+	 */
+	Assert(fdwxact->indoubt ||
+		   fdwxact->status == FDWXACT_STATUS_PREPARED);
+
+	/*
+	 * If the local transaction is already committed, commit prepared
+	 * foreign transaction.
+	 */
+	if (TransactionIdDidCommit(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_COMMITTING;
+
+	/*
+	 * If the local transaction is already aborted, abort prepared
+	 * foreign transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+
+
+	/*
+	 * The local transaction is not in progress but the foreign
+	 * transaction is not prepared on the foreign server. This
+	 * can happen when transaction failed after registered this
+	 * entry but before actual preparing on the foreign server.
+	 * So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is
+	 * about to be committed or aborted. This should not happen except for one
+	 * case where the local transaction is prepared and this foreign transaction
+	 * is being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid),
+				 errhint("The local transaction with xid %u might be prepared",
+						 fdwxact->local_xid)));
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * callback function. The 'state' is passed to the callback function. The fate of
+ * foreign transaction must be determined. If foreign transaction is resolved
+ * successfully, remove the FdwXact entry from the shared memory and also
+ * remove the corresponding on-disk file. If failed, the status of FdwXact
+ * entry changes to 'fallback_status' before erroring out.
+ */
+static void
+FdwXactResolveForeignTransaction(FdwXact fdwxact, FdwXactRslvState *state,
+								 FdwXactStatus fallback_status)
+{
+	ForeignServer		*server;
+	ForeignDataWrapper	*fdw;
+	FdwRoutine			*fdw_routine;
+	bool				is_commit;
+
+	Assert(state != NULL);
+	Assert(state->server && state->usermapping && state->fdwxact_id);
+	Assert(fdwxact != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+		elog(ERROR, "cannot resolve foreign transaction whose fate is not determined");
+
+	is_commit = fdwxact->status == FDWXACT_STATUS_COMMITTING;
+	LWLockRelease(FdwXactLock);
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	PG_TRY();
+	{
+		if (is_commit)
+			fdw_routine->CommitForeignTransaction(state);
+		else
+			fdw_routine->RollbackForeignTransaction(state);
+	}
+	PG_CATCH();
+	{
+		/* Back to the fallback status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = fallback_status;
+		LWLockRelease(FdwXactLock);
+
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	elog(DEBUG1, "successfully %s the foreign transaction with xid %u db %u server %u user %u",
+		 is_commit ? "committed" : "rolled back",
+		 fdwxact->local_xid, fdwxact->dbid, fdwxact->serverid,
+		 fdwxact->userid);
+
+	fdwxact->status = FDWXACT_STATUS_RESOLVED;
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Return palloc'd and initialized FdwXactRslvState.
+ */
+static FdwXactRslvState *
+create_fdwxact_state(void)
+{
+	FdwXactRslvState *state;
+
+	state = palloc(sizeof(FdwXactRslvState));
+	state->server = NULL;
+	state->usermapping = NULL;
+	state->fdwxact_id = NULL;
+	state->flags = 0;
+
+	return state;
+}
+
+/*
+ * Return at least one FdwXact entry that matches to given argument,
+ * otherwise return NULL. All arguments must be valid values so that it can
+ * search exactly one (or none) entry. Note that this function intended to be
+ * used for modifying the returned FdwXact entry, so the caller must hold
+ * FdwXactLock in exclusive mode and it doesn't include the in-progress
+ * FdwXact entries.
+ */
+static FdwXact
+get_one_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	/* Include in-dbout transactions but don't include in-progress ones */
+	fdwxact_list = get_fdwxacts(dbid, xid, serverid, userid,
+								true, false, false);
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdwxact_list) <= 1);
+
+	/* Could not find entry */
+	if (fdwxact_list == NIL)
+		return NULL;
+
+	return (FdwXact) linitial(fdwxact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+fdwxact_exists(Oid dbid, Oid serverid, Oid userid)
+{
+	List	*fdwxact_list;
+
+	/* Find entries from all FdwXact entries */
+	fdwxact_list = get_fdwxacts(dbid, InvalidTransactionId, serverid,
+								userid, true, true, true);
+
+	return fdwxact_list != NIL;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_foreign_xacts, and the number of entries to num_p.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't
+ * want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdwxacts(int *num_p)
+{
+	List		*all_fdwxacts;
+	ListCell	*lc;
+	FdwXact		fdwxacts;
+	int			num_fdwxacts = 0;
+
+	Assert(num_p != NULL);
+
+	/* Get all entries */
+	all_fdwxacts = get_fdwxacts(InvalidOid, InvalidTransactionId,
+								InvalidOid, InvalidOid, true,
+								true, true);
+
+	if (all_fdwxacts == NIL)
+	{
+		*num_p = 0;
+		return NULL;
+	}
+
+	fdwxacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdwxacts));
+	*num_p = list_length(all_fdwxacts);
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdwxacts)
+	{
+		FdwXact fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdwxacts + num_fdwxacts, fx,
+			   sizeof(FdwXactData));
+		num_fdwxacts++;
+	}
+
+	list_free(all_fdwxacts);
+
+	return fdwxacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respecitively.
+ */
+static List*
+get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			 bool include_indoubt, bool include_in_progress, bool need_lock)
+{
+	int i;
+	List	*fdwxact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact	fdwxact = FdwXactCtl->fdwxacts[i];
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* include in-doubt transaction? */
+		if (!include_indoubt && fdwxact->indoubt)
+			continue;
+
+		/* include in-progress transaction? */
+		if (!include_in_progress && FdwXactIsBeingResolved(fdwxact))
+			continue;
+
+		/* Append it if matched */
+		fdwxact_list = lappend(fdwxact_list, fdwxact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdwxact_list;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record
+		 * in FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	*id;
+	int		id_len = 0;
+
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		/*
+		 * FDW doesn't provide the callback function, generate an unique
+		 * idenetifier.
+		 */
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+			 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifer \"%s\" is too long",
+						id),
+				 errdetail("foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint'S redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;						/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence fo long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked
+	 * invalid, because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+			  (errmsg_plural("%u foreign transaction state file was written "
+							 "for long-running prepared transactions",
+							 "%u foreign transaction state files were written "
+							 "for long-running prepared transactions",
+							 serialized_fdwxacts,
+							 serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
+									&read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+		   errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, lsn, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not read foreign transaction state from xlog at %X/%X",
+			   (uint32) (lsn >> 32),
+			   (uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+		errmsg("could not recreate foreign transaction state file \"%s\": %m",
+			   path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not write foreign transcation state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			  errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId	origNextXid =
+		XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	*buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+			   errmsg("could not open FDW transaction state file \"%s\": %m",
+					  path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid  != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char		*buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The
+	 * status of the transaction is set as preparing, since we do not
+	 * know the exact status right now. Resolver will set it later
+	 * based on the status of local transaction which prepared this
+	 * foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							  fdwxact_data->serverid, fdwxact_data->userid,
+							  fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know
+	 * the xact status right now. Resolver will set it later based on
+	 * the status of local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;	/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact	fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdwxact(dbid, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		return;
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+		char	*buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(int *newval, void **extra, GucSource source)
+{
+	ForeignTwophaseCommitLevel newForeignTwophaseCommitLevel = *newval;
+
+	/* Parameter check */
+	if (newForeignTwophaseCommitLevel > FOREIGN_TWOPHASE_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+}	WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	7
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdwxacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdwxacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(PG_PREPARED_FDWXACTS_COLS);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "indoubt",
+						   BOOLOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 7, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdwxacts = get_all_fdwxacts(&num_fdwxacts);
+		status->num_xacts = num_fdwxacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdwxact = &status->fdwxacts[status->cur_xact++];
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdwxact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdwxact->dbid);
+		values[1] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[2] = ObjectIdGetDatum(fdwxact->serverid);
+		values[3] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (fdwxact->status)
+		{
+			case FDWXACT_STATUS_INITIAL:
+				xact_status = "initial";
+				break;
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			case FDWXACT_STATUS_RESOLVED:
+				xact_status = "resolved";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		values[5] = BoolGetDatum(fdwxact->indoubt);
+		values[6] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+	FdwXact			fdwxact;
+	FdwXactRslvState	*state;
+	FdwXactStatus		prev_status;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	server = GetForeignServer(serverid);
+	usermapping = GetUserMapping(userid, serverid);
+	state = create_fdwxact_state();
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+	{
+		LWLockRelease(FdwXactLock);
+		PG_RETURN_BOOL(false);
+	}
+
+	state->server = server;
+	state->usermapping = usermapping;
+	state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+
+	SpinLockAcquire(&fdwxact->mutex);
+	prev_status = fdwxact->status;
+	SpinLockRelease(&fdwxact->mutex);
+
+	FdwXactDetermineTransactionFate(fdwxact, false);
+
+	LWLockRelease(FdwXactLock);
+
+	FdwXactResolveForeignTransaction(fdwxact, state, prev_status);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId	xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid				serverid = PG_GETARG_OID(1);
+	Oid				userid = PG_GETARG_OID(2);
+	FdwXact			fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000000..041ac3871f
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,644 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to retry resolution.
+ */
+void
+FdwXactLauncherRequestToLaunchForRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int	slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			resolver->last_resolved_time = 0;
+			resolver->latch = NULL;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int	save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz	last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz	now;
+		long	wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int		rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a foreign_xact_resolution_retry_interval
+		 * but always starts when the backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested
+			 * but not running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we
+			 * should retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool	found = false;
+	int		i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the
+	 * same database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find FdwXact
+		 * entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int unused_slot = -1;
+	int i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign trasanction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to wait
+	 * until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	*resolver_dbs;	/* DBs resolver's running on */
+	HTAB	*fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL	ctl;
+	HASH_SEQ_STATUS status;
+	Oid		*entry;
+	bool	launched;
+	int		i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->indoubt)
+			continue;
+
+		hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+		return false;
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	return launched;
+}
+
+/*
+ * FdwXactLauncherRegister
+ *		Register a background worker running the foreign transaction
+ *      launcher.
+ */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign trasaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		 /* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+						WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+						10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Returns activity of all foreign transaction resolvers.
+ */
+Datum
+pg_stat_get_foreign_xact(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver	*resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t	pid;
+		Oid		dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000000..2c41e58b9e
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,343 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int foreign_xact_resolution_retry_interval;
+int foreign_xact_resolver_timeout = 60 * 1000;
+bool foreign_xact_resolve_indoubt_xacts;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int		save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	MyFdwXactResolver->last_resolved_time = 0;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		PGPROC			*waiter = NULL;
+		TransactionId	waitXid = InvalidTransactionId;
+		TimestampTz		resolutionTs = -1;
+		int			rc;
+		TimestampTz	now;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or got the waiter
+		 * that has future resolution time.
+		 */
+		while ((waiter = FdwXactGetWaiter(&resolutionTs, &waitXid)) != NULL)
+		{
+			CHECK_FOR_INTERRUPTS();
+			Assert(TransactionIdIsValid(waitXid));
+
+			if	(resolutionTs > now)
+				break;
+
+			elog(DEBUG2, "resolver got one waiter with xid %u", waitXid);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveTransaction(MyDatabaseId, waitXid, waiter);
+			CommitTransactionCommand();
+
+			/* Update my stats */
+			SpinLockAcquire(&(MyFdwXactResolver->mutex));
+			MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+			SpinLockRelease(&(MyFdwXactResolver->mutex));
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz last_resolved_time;
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	last_resolved_time = MyFdwXactResolver->last_resolved_time;
+	timeout = TimestampTzPlusMilliseconds(last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * checked FdwXactWaiterExists.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long	sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long	sec_to_timeout;
+		int		microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index f88d72fd86..982c1a36cc 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -13,6 +13,7 @@ OBJS = \
 	clogdesc.o \
 	committsdesc.o \
 	dbasedesc.o \
+	fdwxactdesc.o \
 	genericdesc.o \
 	gindesc.o \
 	gistdesc.o \
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000000..fe0cef9472
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxactdesc.c
+ *		PostgreSQL global transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdwxactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdwxact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		FdwXactOnDiskData *fdwxact_insert = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_insert->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_insert->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_insert->dbid);
+		appendStringInfo(buf, " local xid: %u,", fdwxact_insert->local_xid);
+		appendStringInfo(buf, " id: %s", fdwxact_insert->fdwxact_id);
+	}
+	else
+	{
+		xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_remove->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_remove->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_remove->dbid);
+		appendStringInfo(buf, " local xid: %u", fdwxact_remove->xid);
+	}
+
+}
+
+const char *
+fdwxact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDWXACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDWXACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 1cd97852e8..ea045174e0 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 58091f6b52..200cf9d067 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -10,6 +10,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 6f7ee0c947..80d0972209 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -850,6 +851,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 	return result;
 }
 
+/*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
 /*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
@@ -2262,6 +2292,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2321,6 +2357,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 017f03b6d8..2722d532d7 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1218,6 +1219,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1226,6 +1228,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1264,12 +1267,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1427,6 +1431,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2086,6 +2098,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2246,6 +2261,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2333,6 +2349,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2527,6 +2545,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2732,6 +2751,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7f4f784c0e..96ad310765 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
@@ -5245,6 +5246,7 @@ BootStrapXLOG(void)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6184,6 +6186,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_wal_senders",
 									 max_wal_senders,
 									 ControlFile->max_wal_senders);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
@@ -6724,14 +6729,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -6923,7 +6929,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7419,6 +7428,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7749,6 +7759,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9024,6 +9037,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9457,8 +9471,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9476,6 +9492,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9492,6 +9509,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9697,6 +9715,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -9896,6 +9915,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index c9e75f4370..d6b0159128 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -332,6 +332,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
@@ -824,6 +827,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_foreign_xact AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_foreign_xact() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 40a8ec1abd..408a5085f0 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2858,8 +2858,14 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
+	}
 
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index f197869752..767dbcb3db 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1101,6 +1103,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1419,6 +1433,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
+	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (fdwxact_exists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
 	/*
 	 * Do the deletion
 	 */
@@ -1572,6 +1595,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index c13b1d3501..1dc61fbdea 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/table.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
@@ -937,7 +938,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 513471ab9b..29f376e48c 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,9 +226,31 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
+	}
 
 	return scanstate;
 }
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index c0a15c3412..a9d223a534 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -549,6 +551,10 @@ ExecInsert(ModifyTableState *mtstate,
 										   NULL,
 										   specToken);
 
+			/* Make note that we've wrote on non-temprary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -777,6 +783,10 @@ ldelete:;
 									&tmfd,
 									changingPart);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -1323,6 +1333,10 @@ lreplace:;
 									true /* wait for commit */ ,
 									&tmfd, &lockmode, &update_indexes);
 
+		/* Make note that we've wrote on non-temprary relation */
+		if (RelationNeedsWAL(resultRelationDesc))
+			MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 		switch (result)
 		{
 			case TM_SelfModified:
@@ -2382,6 +2396,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index 61e48ca3f8..15ec09e3ba 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMappingOid - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,20 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction &&
+		 !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction &&
+		 routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data-wrapper must support both commit and rollback routine or either");
+
+	if (routine->PrepareForeignTransaction &&
+		(!routine->CommitForeignTransaction ||
+		 !routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 75fc0d5d33..c8b38142a5 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -12,6 +12,8 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -128,6 +130,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 51c486bebd..616d96215d 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3645,6 +3645,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3848,6 +3854,11 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDWXACT:
+			event_name = "FdwXact";
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -4066,6 +4077,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 7a92dac525..8f63bf6999 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -909,6 +911,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -984,12 +990,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index 5e1dc8a651..c77ca40e1c 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -151,6 +151,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 427b0d59cd..55609eed81 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(void)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(void)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index c3adb2e6c0..ebe35c4788 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -93,6 +93,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -248,6 +250,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1312,6 +1315,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1377,6 +1381,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1426,6 +1431,15 @@ GetOldestXmin(Relation rel, int flags)
 		NormalTransactionIdPrecedes(replication_slot_xmin, result))
 		result = replication_slot_xmin;
 
+	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
 	/*
 	 * After locks have been released and vacuum_defer_cleanup_age has been
 	 * applied, check whether we need to back up further to make logical
@@ -3128,6 +3142,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations fo the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index db47843229..adb276370c 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -49,3 +49,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 CLogTruncationLock					44
+FdwXactLock							45
+FdwXactResolverLock					46
+FdwXactResolutionLock				47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 32df8c85a1..623c87bb6a 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -421,6 +422,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* Initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -822,6 +827,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 0a6f80963b..9b40134aa4 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3030,6 +3032,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 9f179a9129..4a0fcf397c 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -404,6 +405,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 	{NULL, 0, false}
 };
 
+/*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
 /*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
@@ -731,6 +751,12 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
+	/* FDWXACT_RESOLVER */
+	gettext_noop("Foreign Transaction Management / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2413,6 +2439,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FDWXACT_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FDWXACT_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4456,6 +4528,16 @@ static struct config_enum ConfigureNamesEnum[] =
 		NULL, assign_synchronous_commit, NULL
 	},
 
+	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
 	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index e1048c0047..859f32eb95 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -342,6 +344,20 @@
 #max_sync_workers_per_subscription = 2	# taken from max_logical_replication_workers
 
 
+#------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
 #------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index a0b0458108..8701c5f005 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 7f1534aebb..2195fb6e90 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -208,6 +208,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 19e21ab491..9ae3bfe4dd 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -301,6 +301,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index f9cfeae264..a5f2aa1a09 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -710,6 +710,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -914,6 +915,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/fdwxactdesc.c b/src/bin/pg_waldump/fdwxactdesc.c
new file mode 120000
index 0000000000..ce8c21880c
--- /dev/null
+++ b/src/bin/pg_waldump/fdwxactdesc.c
@@ -0,0 +1 @@
+../../../src/backend/access/rmgrdesc/fdwxactdesc.c
\ No newline at end of file
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca4b1..b616cea347 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000000..5519bd908f
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,164 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL global transaction manager
+ *
+ * Portions Copyright (c) 2018, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDWXACT_H
+#define FDWXACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* fdwXactState */
+#define	FDWXACT_NOT_WAITING		0
+#define	FDWXACT_WAITING			1
+#define	FDWXACT_WAIT_COMPLETE	2
+
+/* Flag passed to FDW transaction management APIs */
+#define FDWXACT_FLAG_ONEPHASE		0x01	/* transaction can commit/rollback
+											   without preparation */
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER,		/* use twophase commit where available */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support
+										   twophase commit */
+} ForeignTwophaseCommitLevel;
+
+/* Enum to track the status of foreign transaction */
+typedef enum
+{
+	FDWXACT_STATUS_INVALID,
+	FDWXACT_STATUS_INITIAL,
+	FDWXACT_STATUS_PREPARING,		/* foreign transaction is being prepared */
+	FDWXACT_STATUS_PREPARED,		/* foreign transaction is prepared */
+	FDWXACT_STATUS_COMMITTING,		/* foreign prepared transaction is to
+									 * be committed */
+	FDWXACT_STATUS_ABORTING,		/* foreign prepared transaction is to be
+									 * aborted */
+	FDWXACT_STATUS_RESOLVED
+} FdwXactStatus;
+
+typedef struct FdwXactData *FdwXact;
+
+/*
+ * Shared memory state of a single foreign transaction.
+ */
+typedef struct FdwXactData
+{
+	FdwXact			fdwxact_free_next;	/* Next free FdwXact entry */
+
+	Oid				dbid;			/* database oid where to find foreign server
+									 * and user mapping */
+	TransactionId	local_xid;		/* XID of local transaction */
+	Oid				serverid;		/* foreign server where transaction takes
+									 * place */
+	Oid				userid;			/* user who initiated the foreign
+									 * transaction */
+	Oid				umid;
+	bool			indoubt;		/* Is an in-doubt transaction? */
+	slock_t			mutex;			/* Protect the above fields */
+
+	/* The status of the foreign transaction, protected by FdwXactLock */
+	FdwXactStatus 	status;
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;		/* XLOG offset of inserting this entry start */
+	XLogRecPtr	insert_end_lsn;		/* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN];		/* prepared transaction identifier */
+} FdwXactData;
+
+/*
+ * Shared memory layout for maintaining foreign prepared transaction entries.
+ * Adding or removing FdwXact entry needs to hold FdwXactLock in exclusive mode,
+ * and iterating fdwXacts needs that in shared mode.
+ */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		free_fdwxacts;
+
+	/* Number of valid foreign transaction entries */
+	int			num_fdwxacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdwxacts[FLEXIBLE_ARRAY_MEMBER];		/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* State data for foreign transaction resolution, passed to FDW callbacks */
+typedef struct FdwXactRslvState
+{
+	/* Foreign transaction information */
+	char	*fdwxact_id;
+
+	ForeignServer	*server;
+	UserMapping		*usermapping;
+
+	int		flags;			/* OR of FDWXACT_FLAG_xx flags */
+} FdwXactRslvState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+/* Function declarations */
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern bool fdwxact_exists(Oid dboid, Oid serverid, Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern void PreCommit_FdwXacts(void);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+extern void FdwXactResolveTransaction(Oid dbid, TransactionId xid, PGPROC *waiter);
+extern bool FdwXactResolveInDoubtTransactions(Oid dbid);
+extern PGPROC *FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p);
+extern void FdwXactCleanupAtProcExit(void);
+extern void RegisterFdwXactByRelId(Oid relid, bool modified);
+extern void RegisterFdwXactByServerId(Oid serverid, bool modified);
+extern void FdwXactMarkForeignServerAccessed(Oid relid, bool modified);
+extern bool check_foreign_twophase_commit(int *newval, void **extra,
+										  GucSource source);
+extern bool FdwXactWaiterExists(Oid dbid);
+
+#endif   /* FDWXACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000000..dd0f5d16ff
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,29 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherRequestToLaunch(void);
+extern void FdwXactLauncherRequestToLaunchForRetry(void);
+extern void FdwXactLaunchOrWakeupResolver(void);
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+
+
+#endif	/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000000..2607654024
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int foreign_xact_resolver_timeout;
+
+#endif		/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000000..39ca66beef
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,54 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDWXACT_INSERT	0x00
+#define XLOG_FDWXACT_REMOVE	0x10
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDWXACT_ID_MAX_LEN 200
+
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdwxact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+	bool		force;
+} xl_fdwxact_remove;
+
+extern void fdwxact_redo(XLogReaderState *record);
+extern void fdwxact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdwxact_identify(uint8 info);
+
+#endif	/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000000..55fc970b69
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,66 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/resovler_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t	pid;	/* this resolver's PID, or 0 if not active */
+	Oid		dbid;	/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool	in_use;
+
+	/* Stats */
+	TimestampTz	last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t	mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	*latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/* Foreign transaction resolution queue. Protected by FdwXactLock */
+	SHM_QUEUE	fdwxact_queue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch		*launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+#define SizeOfFdwXactRslvCtlData \
+	(offsetof(FdwXactRslvCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif	/* RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index c88dccfb8d..254a663b4d 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Foreign Transactions", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 2ca71c3445..bd027a2861 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index a04fc70326..6f1f336e31 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -108,6 +108,13 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
+/*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 087918d41d..ed4d08b4af 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -232,6 +232,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index de5670e538..9884f5f8e7 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index fcf2a1214c..db4fa89699 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5191,6 +5191,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '9705', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_foreign_xact', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_foreign_xact' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5904,6 +5911,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '9706', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o,o}',
+  proargnames => '{dbid,xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '9707', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '9708', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -6022,6 +6047,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '9709',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 95556dfb15..8d046cc4e4 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 5e0cf533fb..5596ee591c 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -69,6 +69,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 											   bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 														 bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index aecb6013f0..1c6cd15652 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -776,6 +776,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -853,7 +855,9 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDWXACT,
+	WAIT_EVENT_FDWXACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -934,6 +938,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index d21780108b..35ffbbca93 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -16,6 +16,7 @@
 
 #include "access/clog.h"
 #include "access/xlogdefs.h"
+#include "datatype/timestamp.h"
 #include "lib/ilist.h"
 #include "storage/latch.h"
 #include "storage/lock.h"
@@ -152,6 +153,16 @@ struct PGPROC
 	int			syncRepState;	/* wait state for sync rep */
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
+	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
 	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index a5c7d0c064..0f73b64937 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -125,4 +127,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 454c2df487..6010dbcdee 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,9 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
+	FDWXACT_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 70e1e2f78d..dbb7f18e8c 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1341,6 +1341,14 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.dbid,
+    f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(dbid, xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
@@ -1846,6 +1854,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_foreign_xact| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_foreign_xact() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_gssapi| SELECT s.pid,
     s.gss_auth AS gss_authenticated,
     s.gss_princ AS principal,
-- 
2.23.0

From ccf30999cad0ddef2a06437890d0be1c3cdf6059 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 5 Dec 2019 17:01:08 +0900
Subject: [PATCH v27 3/5] Documentation update.

Original Author: Masahiko Sawada <sawada.mshk@gmail.com>
---
 doc/src/sgml/catalogs.sgml                | 145 +++++++++++++
 doc/src/sgml/config.sgml                  | 146 ++++++++++++-
 doc/src/sgml/distributed-transaction.sgml | 158 +++++++++++++++
 doc/src/sgml/fdwhandler.sgml              | 236 ++++++++++++++++++++++
 doc/src/sgml/filelist.sgml                |   1 +
 doc/src/sgml/func.sgml                    |  89 ++++++++
 doc/src/sgml/monitoring.sgml              |  60 ++++++
 doc/src/sgml/postgres.sgml                |   1 +
 doc/src/sgml/storage.sgml                 |   6 +
 9 files changed, 841 insertions(+), 1 deletion(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 85ac79f07e..95695e0374 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8267,6 +8267,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
       <entry>open cursors</entry>
      </row>
 
+     <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
      <row>
       <entry><link linkend="view-pg-file-settings"><structname>pg_file_settings</structname></link></entry>
       <entry>summary of configuration file contents</entry>
@@ -9717,6 +9722,146 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with that this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on that the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>initial</literal> : Initial status.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>resolved</literal> : This foreign transaction has been resolved.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-dbout status and
+       needs to be resolved by calling <function>pg_resolve_fdwxact</function>
+       function.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index e07dc01e80..7aebd8adf9 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4398,7 +4398,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -8863,6 +8862,151 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign
+         transaction to be resolved before the command returns a "success"
+         indication to the client. Valid values are <literal>required</literal>,
+         <literal>prefer</literal> and <literal>disabled</literal>. The default
+         setting is <literal>disabled</literal>. Setting to
+         <literal>disabled</literal> don't use two-phase commit protocol to
+         commit or rollback distributed transactions. When set to
+         <literal>required</literal> the distributed transaction strictly
+         requires that all written servers can use two-phase commit protocol.
+         That is, the distributed transaction cannot commit if even one server
+         does not support the transaction management callback routines
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. Note that when <literal>disabled</literal> or
+         <literal>prefer</literal> there can be risk of database consistency
+         among all servers that involved in the distributed transaction when some
+         foreign server crashes during committing the distributed transaction.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero
+         value to set this parameter either <literal>required</literal> or
+         <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000000..350b1afe68
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,158 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).
+    A <productname>PostgreSQL</productname> server that received SQL is called
+    <firstterm>coordinator node</firstterm> who is responsible for coordinating
+    all the partipanting transactions. Using two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed
+    transaction manager prepares all transaction on the foreign servers if
+    two-phase commit is required. Two-phase commit is required when the
+    transaction modifies data on two or more servers including the local server
+    itself and <xref linkend="guc-foreign-twophase-commit"/>is
+    <literal>required</literal> or <literal>prefer</literal>. If all preparations
+    on foreign servers got successful go to the next step. Any failure happens
+    in this step <productname>PostgreSQL</productname> changes to rollback, then
+    rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the
+    transaction locally. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes rollback, then rollback all
+    transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolution">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that is
+    responsible for foreign transaction resolution. They commit or rollback all
+    prepared transaction on foreign servers if the coordinator received agreement
+    messages from all foreign servers during the first step.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on one database of the coordinator side. On failure during resolution, they
+    retries to resolve at an interval of
+    <varname>foreign_transaction_resolution_interval</varname> time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped. So to drop the database, you can call
+     <function>pg_stop_foreign_xact_resovler</function> function before dropping
+     the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>Manual Resolution of In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, distributed transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the coordinator node crashed during either preparing or
+    resolving distributed transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="pg-stat-foreign-xact-view"/>
+    view. These foreign transactions need to be resolved by using
+    <function>pg_resolve_foriegn_xact</function> function.
+    <productname>PostgreSQL</productname> doesn't have facilities to automatially
+    resolve in-doubt transactions. These behavior might change in a future release.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-monitoring">
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-foreign-xact-view"><literal>pg_stat_foreign_xact</literal></link>
+    view. This view contains one row for every foreign transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 6587678af2..dd0358ef22 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1415,6 +1415,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1894,4 +2015,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 3da2365ea9..80a87fa5d1 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 6c4359dc7b..35b6fbb319 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -21766,6 +21766,95 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 0bfd6151c4..455d423d9c 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -376,6 +376,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_foreign_xact</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1244,6 +1252,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry><literal>CheckpointerMain</literal></entry>
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
+        <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
         <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
@@ -1467,6 +1487,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry><literal>SyncRep</literal></entry>
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
+        <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
         <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
@@ -2371,6 +2395,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-view" xreflabel="pg_stat_foreign_xact">
+   <title><structname>pg_stat_foreign_xact</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index e59cba7997..dee3f72f7e 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -163,6 +163,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 1c19e863d2..3f4c806ed1 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -83,6 +83,12 @@ Item
   subsystem</entry>
 </row>
 
+<row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
 <row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
-- 
2.23.0

From d30099d21190763a9f546a595d4aba338a8dc3aa Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 5 Dec 2019 17:01:15 +0900
Subject: [PATCH v27 4/5] postgres_fdw supports atomic commit APIs.

Original Author: Masahiko Sawada <sawada.mshk@gmail.com>
---
 contrib/postgres_fdw/Makefile                 |   7 +-
 contrib/postgres_fdw/connection.c             | 603 +++++++++++-------
 .../postgres_fdw/expected/postgres_fdw.out    | 265 +++++++-
 contrib/postgres_fdw/fdwxact.conf             |   3 +
 contrib/postgres_fdw/postgres_fdw.c           |  21 +-
 contrib/postgres_fdw/postgres_fdw.h           |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql     | 120 +++-
 doc/src/sgml/postgres-fdw.sgml                |  45 ++
 8 files changed, 822 insertions(+), 249 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index ee8a80a392..91fa6e39fc 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -16,7 +16,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -29,3 +29,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index 29c811a80b..30b9815ecc 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2020, PostgreSQL Global Development Group
  *
@@ -12,6 +12,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "catalog/pg_user_mapping.h"
@@ -55,6 +56,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -68,17 +70,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 								   SubTransactionId mySubid,
 								   SubTransactionId parentSubid,
@@ -91,23 +89,26 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 									 PGresult **result);
 static bool UserMappingPasswordRequired(UserMapping *user);
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -127,7 +128,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -135,12 +135,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -154,6 +148,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -181,6 +190,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -189,6 +199,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -199,6 +210,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -206,11 +226,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -438,7 +486,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -665,193 +713,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 	PG_END_TRY();
 }
 
-/*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow any remote transactions, since it's not
-					 * very reasonable to hold them open until the prepared
-					 * transaction is committed.  For the moment, throw error
-					 * unconditionally; later we might allow read-only cases.
-					 * Note that the error will cause us to come right back
-					 * here with event == XACT_EVENT_ABORT, so we'll clean up
-					 * the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot PREPARE a transaction that has operated on postgres_fdw foreign tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
 /*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
@@ -868,10 +729,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -882,6 +739,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1216,3 +1077,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recusively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 84fd3ad2e0..15cb1d1ca7 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -191,15 +210,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8930,16 +8951,226 @@ DROP OWNED BY regress_nosuper;
 DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
--- Two-phase transactions are not supported.
+
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
 BEGIN;
-SELECT count(*) FROM ft1;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
  count 
 -------
-   822
+     0
 (1 row)
 
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
-ERROR:  cannot PREPARE a transaction that has operated on postgres_fdw foreign tables
-ROLLBACK;
-WARNING:  there is no transaction in progress
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000000..3fdbf93cdb
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 2175dff824..0873d1d4b7 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include <limits.h>
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -504,7 +505,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 							  const PgFdwRelationInfo *fpinfo_o,
 							  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -558,6 +558,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1434,7 +1439,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2372,7 +2377,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2746,7 +2751,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3566,7 +3571,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4441,7 +4446,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4527,7 +4532,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4755,7 +4760,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index eef410db39..43ffd4f73f 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "libpq-fe.h"
@@ -129,7 +130,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -137,6 +138,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 							   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int	ExtractConnectionOptions(List *defelems,
@@ -203,6 +207,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 									bool is_subquery,
 									List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index acd7275c72..562b59056e 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2608,9 +2631,98 @@ DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
 
--- Two-phase transactions are not supported.
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
 BEGIN;
-SELECT count(*) FROM ft1;
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
+INSERT INTO ft7_2pc VALUES(1);
 ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 94992be427..3f52daa11e 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -477,6 +477,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -504,6 +541,14 @@
    managed by creating corresponding remote savepoints.
   </para>
 
+  <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
   <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
-- 
2.23.0

From 355e6c7f38108b24c42c0bb4420e5f8a37c0f24e Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 5 Dec 2019 17:01:15 +0900
Subject: [PATCH v18 4/5] postgres_fdw supports atomic commit APIs.

Authors: Masahiko Sawada, Ashutosh Bapat
---
 contrib/postgres_fdw/Makefile                 |   7 +-
 contrib/postgres_fdw/connection.c             | 603 +++++++++++-------
 .../postgres_fdw/expected/postgres_fdw.out    | 265 +++++++-
 contrib/postgres_fdw/fdwxact.conf             |   3 +
 contrib/postgres_fdw/postgres_fdw.c           |  21 +-
 contrib/postgres_fdw/postgres_fdw.h           |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql     | 120 +++-
 doc/src/sgml/postgres-fdw.sgml                |  45 ++
 8 files changed, 822 insertions(+), 249 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index ee8a80a392..91fa6e39fc 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -16,7 +16,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -29,3 +29,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index 29c811a80b..30b9815ecc 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2020, PostgreSQL Global Development Group
  *
@@ -12,6 +12,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "catalog/pg_user_mapping.h"
@@ -55,6 +56,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -68,17 +70,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 								   SubTransactionId mySubid,
 								   SubTransactionId parentSubid,
@@ -91,23 +89,26 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 									 PGresult **result);
 static bool UserMappingPasswordRequired(UserMapping *user);
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -127,7 +128,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -135,12 +135,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -154,6 +148,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -181,6 +190,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -189,6 +199,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -199,6 +210,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -206,11 +226,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -438,7 +486,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -665,193 +713,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 	PG_END_TRY();
 }
 
-/*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow any remote transactions, since it's not
-					 * very reasonable to hold them open until the prepared
-					 * transaction is committed.  For the moment, throw error
-					 * unconditionally; later we might allow read-only cases.
-					 * Note that the error will cause us to come right back
-					 * here with event == XACT_EVENT_ABORT, so we'll clean up
-					 * the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot PREPARE a transaction that has operated on postgres_fdw foreign tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
 /*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
@@ -868,10 +729,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -882,6 +739,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1216,3 +1077,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recusively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 62c2697920..cd871fe314 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -191,15 +210,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8961,16 +8982,226 @@ DROP OWNED BY regress_nosuper;
 DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
--- Two-phase transactions are not supported.
+
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
 BEGIN;
-SELECT count(*) FROM ft1;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
  count 
 -------
-   822
+     0
 (1 row)
 
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
-ERROR:  cannot PREPARE a transaction that has operated on postgres_fdw foreign tables
-ROLLBACK;
-WARNING:  there is no transaction in progress
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000000..3fdbf93cdb
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 2175dff824..0873d1d4b7 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include <limits.h>
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -504,7 +505,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 							  const PgFdwRelationInfo *fpinfo_o,
 							  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -558,6 +558,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1434,7 +1439,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2372,7 +2377,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2746,7 +2751,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3566,7 +3571,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4441,7 +4446,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4527,7 +4532,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4755,7 +4760,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index eef410db39..43ffd4f73f 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "libpq-fe.h"
@@ -129,7 +130,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -137,6 +138,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 							   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int	ExtractConnectionOptions(List *defelems,
@@ -203,6 +207,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 									bool is_subquery,
 									List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 83971665e3..ce5785c27a 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2628,9 +2651,98 @@ DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
 
--- Two-phase transactions are not supported.
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
 BEGIN;
-SELECT count(*) FROM ft1;
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
+INSERT INTO ft7_2pc VALUES(1);
 ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 94992be427..3f52daa11e 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -477,6 +477,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -504,6 +541,14 @@
    managed by creating corresponding remote savepoints.
   </para>
 
+  <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
   <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
-- 
2.23.0

From 4515ea6d6d128658ce845410a7baa1d258254675 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 5 Dec 2019 17:01:26 +0900
Subject: [PATCH v18 5/5] Add regression tests for atomic commit.

Authors: Masahiko Sawada, Ashtosh Bapat
---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index fa8e031526..d47d96975b 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000000..9af9bb81dc
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 9a4e52bc7b..4ab217dc98 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2319,9 +2319,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2336,7 +2339,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.23.0

From d3b077ca06ce6a56a1eea16d08f615e53f25d64e Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 5 Dec 2019 17:01:08 +0900
Subject: [PATCH v18 3/5] Documentation update.

Authors: Masahiko Sawada, Ashutosh Bapat
---
 doc/src/sgml/catalogs.sgml                | 145 +++++++++++++
 doc/src/sgml/config.sgml                  | 146 ++++++++++++-
 doc/src/sgml/distributed-transaction.sgml | 158 +++++++++++++++
 doc/src/sgml/fdwhandler.sgml              | 236 ++++++++++++++++++++++
 doc/src/sgml/filelist.sgml                |   1 +
 doc/src/sgml/func.sgml                    |  89 ++++++++
 doc/src/sgml/monitoring.sgml              |  60 ++++++
 doc/src/sgml/postgres.sgml                |   1 +
 doc/src/sgml/storage.sgml                 |   6 +
 9 files changed, 841 insertions(+), 1 deletion(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index a10b66569b..23570e9f2c 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8155,6 +8155,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
       <entry>open cursors</entry>
      </row>
 
+     <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
      <row>
       <entry><link linkend="view-pg-file-settings"><structname>pg_file_settings</structname></link></entry>
       <entry>summary of configuration file contents</entry>
@@ -9613,6 +9618,146 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with that this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on that the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>initial</literal> : Initial status.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>resolved</literal> : This foreign transaction has been resolved.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-dbout status and
+       needs to be resolved by calling <function>pg_resolve_fdwxact</function>
+       function.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index c1128f89ec..2dde3185f6 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4403,7 +4403,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -8868,6 +8867,151 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign
+         transaction to be resolved before the command returns a "success"
+         indication to the client. Valid values are <literal>required</literal>,
+         <literal>prefer</literal> and <literal>disabled</literal>. The default
+         setting is <literal>disabled</literal>. Setting to
+         <literal>disabled</literal> don't use two-phase commit protocol to
+         commit or rollback distributed transactions. When set to
+         <literal>required</literal> the distributed transaction strictly
+         requires that all written servers can use two-phase commit protocol.
+         That is, the distributed transaction cannot commit if even one server
+         does not support the transaction management callback routines
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. Note that when <literal>disabled</literal> or
+         <literal>prefer</literal> there can be risk of database consistency
+         among all servers that involved in the distributed transaction when some
+         foreign server crashes during committing the distributed transaction.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero
+         value to set this parameter either <literal>required</literal> or
+         <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000000..350b1afe68
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,158 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).
+    A <productname>PostgreSQL</productname> server that received SQL is called
+    <firstterm>coordinator node</firstterm> who is responsible for coordinating
+    all the partipanting transactions. Using two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed
+    transaction manager prepares all transaction on the foreign servers if
+    two-phase commit is required. Two-phase commit is required when the
+    transaction modifies data on two or more servers including the local server
+    itself and <xref linkend="guc-foreign-twophase-commit"/>is
+    <literal>required</literal> or <literal>prefer</literal>. If all preparations
+    on foreign servers got successful go to the next step. Any failure happens
+    in this step <productname>PostgreSQL</productname> changes to rollback, then
+    rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the
+    transaction locally. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes rollback, then rollback all
+    transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolution">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that is
+    responsible for foreign transaction resolution. They commit or rollback all
+    prepared transaction on foreign servers if the coordinator received agreement
+    messages from all foreign servers during the first step.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on one database of the coordinator side. On failure during resolution, they
+    retries to resolve at an interval of
+    <varname>foreign_transaction_resolution_interval</varname> time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped. So to drop the database, you can call
+     <function>pg_stop_foreign_xact_resovler</function> function before dropping
+     the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>Manual Resolution of In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, distributed transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the coordinator node crashed during either preparing or
+    resolving distributed transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="pg-stat-foreign-xact-view"/>
+    view. These foreign transactions need to be resolved by using
+    <function>pg_resolve_foriegn_xact</function> function.
+    <productname>PostgreSQL</productname> doesn't have facilities to automatially
+    resolve in-doubt transactions. These behavior might change in a future release.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-monitoring">
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-foreign-xact-view"><literal>pg_stat_foreign_xact</literal></link>
+    view. This view contains one row for every foreign transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 6587678af2..dd0358ef22 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1415,6 +1415,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1894,4 +2015,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 3da2365ea9..80a87fa5d1 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index ceda48e0fc..eaf800621f 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -21803,6 +21803,95 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 87586a7b06..e107c9599e 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -376,6 +376,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_foreign_xact</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1256,6 +1264,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry><literal>CheckpointerMain</literal></entry>
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
+        <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
         <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
@@ -1483,6 +1503,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry><literal>SyncRep</literal></entry>
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
         </row>
+        <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
         <row>
          <entry morerows="2"><literal>Timeout</literal></entry>
          <entry><literal>BaseBackupThrottle</literal></entry>
@@ -2391,6 +2415,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-view" xreflabel="pg_stat_foreign_xact">
+   <title><structname>pg_stat_foreign_xact</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index e59cba7997..dee3f72f7e 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -163,6 +163,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 1c19e863d2..3f4c806ed1 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -83,6 +83,12 @@ Item
   subsystem</entry>
 </row>
 
+<row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
 <row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
-- 
2.23.0

From 7b8bad2a26a0d5b02b10a1540b4de2c32fdc2047 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 5 Dec 2019 16:59:47 +0900
Subject: [PATCH v18 1/5] Keep track of writing on non-temporary relation

Authors: Masahiko Sawada, Ahutosh Bapat
---
 src/backend/executor/nodeModifyTable.c | 16 ++++++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 22 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index d71c0a4322..870a7428f1 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -574,6 +574,10 @@ ExecInsert(ModifyTableState *mtstate,
 										   NULL,
 										   specToken);
 
+			/* Make note that we've wrote on non-temporary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -612,6 +616,10 @@ ExecInsert(ModifyTableState *mtstate,
 							   estate->es_output_cid,
 							   0, NULL);
 
+			/* Make note that we've wrote on non-temporary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -963,6 +971,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temporary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1475,6 +1487,10 @@ lreplace:;
 	if (canSetTag)
 		(estate->es_processed)++;
 
+	/* Make note that we've wrote on non-temporary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/* AFTER ROW UPDATE Triggers */
 	ExecARUpdateTriggers(estate, resultRelInfo, tupleid, oldtuple, slot,
 						 recheckIndexes,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7ee04babc2..a04fc70326 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -102,6 +102,12 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
-- 
2.23.0

From 311fd76e036e7927310567730b108ad1d25425b7 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 5 Dec 2019 17:00:50 +0900
Subject: [PATCH v18 2/5] Support atomic commit among multiple foreign servers.

Authors: Masahiko Sawada, Ashutosh Bapat
---
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  130 +
 src/backend/access/fdwxact/fdwxact.c          | 2832 +++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  641 ++++
 src/backend/access/fdwxact/resolver.c         |  343 ++
 src/backend/access/rmgrdesc/Makefile          |    1 +
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   58 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/copy.c                   |    6 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    8 +
 src/backend/executor/nodeForeignscan.c        |   24 +
 src/backend/executor/nodeModifyTable.c        |    6 +
 src/backend/foreign/foreign.c                 |   55 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   20 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   82 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/fdwxactdesc.c              |    1 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  167 +
 src/include/access/fdwxact_launcher.h         |   29 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   54 +
 src/include/access/resolver_internal.h        |   66 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   29 +
 src/include/foreign/fdwapi.h                  |   12 +
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    9 +-
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    3 +
 src/test/regress/expected/rules.out           |   13 +
 55 files changed, 4916 insertions(+), 18 deletions(-)
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100644 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 120000 src/bin/pg_waldump/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a8bb..49480dd039 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000000..0207a66fb4
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000000..c20570022c
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,130 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a consistent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+---------------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consists
+of the following four steps:
+
+1. Foreign Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant. The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+we record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE all foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared transaction on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step (commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itself to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rollbacked the resolver process wake up the waiter.
+
+
+Foreign Data Wrapper Callbacks for Transaction Management
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForeignTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for either PREPARE, COMMIT or
+ROLLBACK the transaction on the foreign server respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transaction in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions has an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_INITIAL
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_PREPARING, FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING
+before the foreign transaction is prepared, committed and aborted by FDW
+callback functions respectively(*1). And the status then changes to
+FDWXACT_STATUS_RESOLVED once the foreign transaction are resolved, and then
+the corresponding FdwXact entry is removed with WAL logging. If failed during
+processing foreign transaction (e.g., preparing, committing or aborting) the
+status changes back to the previous status. Therefore the status
+FDWXACT_STATUS_xxxING appear only during the foreign transaction is being
+processed by an FDW callback function.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*2). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                      INVALID                       |
+ +----------------------------------------------------+
+    |                      |                       |
+    |                      v                       |
+    |           +---------------------+            |
+    |           |       INITIAL       |            |
+    |           +---------------------+            |
+   (*2)                    |                      (*2)
+    |                      v                       |
+    |           +---------------------+            |
+    |           |    PREPARING(*1)    |            |
+    |           +---------------------+            |
+    |                      |                       |
+    v                      v                       v
+ +----------------------------------------------------+
+ |                      PREPARED                      |
+ +----------------------------------------------------+
+           |                               |
+           v                               v
+ +--------------------+          +--------------------+
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |
+ +--------------------+          +--------------------+
+           |                               |
+           v                               v
+ +----------------------------------------------------+
+ |                      RESOLVED                      |
+ +----------------------------------------------------+
+
+(*1) Status that appear only during being processed by FDW
+(*2) Paths for recovered FdwXact entries
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100644
index 0000000000..db2e9a4fe3
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2832 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers atomically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit. The foreign servers are
+ * registered if FDW has both CommitForeignTransaction API and
+ * RollbackForeignTransaction API. Registered participant servers are identified
+ * by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request. We have one queue
+ * of which elements are ordered by the timestamp that they expect to be
+ * processed at. Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp that they expects to be processed.
+ * Similarly if failed to resolve them, it enqueues again with new timestamp
+ * (its timestamp + foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction). Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by
+ * pg_resovle_foreign_xact() function.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or
+ * more servers including itself. In other case, all foreign transactions are
+ * committed or rolled back during pre-commit.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed by FDW, the corresponding
+ * FdwXact entry is update. In order to protect the entry from concurrent
+ * removing we need to hold a lock on the entry or a lock for entire global
+ * array. However, we don't want to hold the lock during FDW is processing the
+ * foreign transaction that may take a unpredictable time. To avoid this, the
+ * in-memory data of foreign transaction follows a locking model based on
+ * four linked concepts:
+ *
+ * * A foreign transaction's status variable is switched using the LWLock
+ *   FdwXactLock, which need to be hold in exclusive mode when updating the
+ *   status, while readers need to hold it in shared mode when looking at the
+ *   status.
+ * * A process who is going to update FdwXact entry cannot process foreign
+ *   transaction that is being resolved.
+ * * So setting the status to FDWXACT_STATUS_PREPARING,
+ *   FDWXACT_STATUS_COMMITTING or FDWXACT_STATUS_ABORTING, which makes foreign
+ *   transaction in-progress states, means to own the FdwXact entry, which
+ *   protect it from updating/removing by concurrent writers.
+ * * Individual fields are protected by mutex where only the backend owning
+ *   the foreign transaction is authorized to update the fields from its own
+ *   one.
+ *
+ * Therefore, before doing PREPARE, COMMIT PREPARED or ROLLBACK PREPARED a
+ * process who is going to call transaction callback functions needs to change
+ * the status to the corresponding status above while holding FdwXactLock in
+ * exclusive mode, and call callback function after releasing the lock.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk. FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon. We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts. If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to disk.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Atomic commit is enabled by configuration */
+#define IsForeignTwophaseCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	(IsForeignTwophaseCommitEnabled() && \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED))
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define IsSeverCapableOfTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/* Check the FdwXact is begin resolved */
+#define FdwXactIsBeingResolved(fx) \
+	(((((FdwXact)(fx))->status) == FDWXACT_STATUS_PREPARING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_COMMITTING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_ABORTING))
+
+/*
+ * Structure to bundle the foreign transaction participant. This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry is
+	 * not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer *server;
+	UserMapping *usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char	   *fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool		modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function prepare_foreign_xact_fn;
+	CommitForeignTransaction_function commit_foreign_xact_fn;
+	RollbackForeignTransaction_function rollback_foreign_xact_fn;
+	GetPrepareId_function get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit. This list
+ * has only foreign servers that provides transaction management callbacks,
+ * that is CommitForeignTransaction and RollbackForeignTransaction.
+ */
+static List *FdwXactParticipants = NIL;
+
+/*
+ * True is the current transaction needs to be committed together with
+ * foreign servers.
+ */
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int			max_prepared_foreign_xacts = 0;
+int			max_foreign_xact_resolvers = 0;
+int			foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid,
+										 FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+												 bool for_commit);
+static void FdwXactResolveForeignTransaction(FdwXact fdwxact,
+											 FdwXactRslvState *state,
+											 FdwXactStatus fallback_status);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid, void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static void FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock);
+static bool is_foreign_twophase_commit_required(bool *local_modified);
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static List *get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						  bool including_indoubts, bool include_in_progress,
+						  bool need_lock);
+static FdwXact get_all_fdwxacts(int *num_p);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXact get_fdwxact_to_resolve(Oid dbid, TransactionId xid);
+static FdwXactRslvState *create_fdwxact_state(void);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation	rel;
+	Oid			serverid;
+	Oid			userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign transaction identified by given server id and user id.
+ * Registered foreign transactions are managed by the global transaction
+ * manager until the end of the transaction.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant *fdw_part;
+	ForeignServer *foreign_server;
+	UserMapping *user_mapping;
+	MemoryContext old_ctx;
+	FdwRoutine *routine;
+	ListCell   *lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Don't register foreign server if it doesn't provide both commit and
+	 * rollback transaction management callbacks and is modified.
+	 */
+	if (!routine->CommitForeignTransaction &&
+		!routine->RollbackForeignTransaction &&
+		modified)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		pfree(routine);
+		return;
+	}
+
+	/*
+	 * Remember we touched the foreign server that is not capable of two-phase
+	 * commit.
+	 */
+	if (!routine->PrepareForeignTransaction)
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+	foreign_server = GetForeignServer(serverid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&(fdwxacts[cnt].mutex));
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * If foreign twophase commit is required, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign server's FDWs to support two-phase commit protocol and ask them to
+ *  prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and for 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_twophase_commit;
+	bool		local_modified;
+	ListCell   *lc = NULL;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * we require all modified server have to be capable of two-phase commit
+	 * protocol.
+	 */
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on foreign server that doesn't support atomic commit")));
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false if
+	 * foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = is_foreign_twophase_commit_required(&local_modified);
+
+	/* Attempt to commit foreign transactions in one-phase */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool		commit = false;
+
+		if (!need_twophase_commit)
+		{
+			/* Can commit in one-phase if two-phase commit is not required */
+			commit = true;
+		}
+		else if (!fdw_part->modified)
+		{
+			/*
+			 * Non-modified foreign transaction always can be committed in
+			 * one-phase regardless of two-phase commit support.
+			 */
+			commit = true;
+		}
+		else if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER &&
+				 !IsSeverCapableOfTwophaseCommit(fdw_part))
+		{
+			/*
+			 * In 'prefer' case, non-twophase-commit capable server can be
+			 * committed in one-phase.
+			 */
+			commit = true;
+		}
+
+		if (commit)
+		{
+			/* Commit the foreign transaction in one-phase */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, true);
+
+			/* Delete it from the participant list */
+			FdwXactParticipants = foreach_delete_current(FdwXactParticipants, lc);
+		}
+	}
+
+	/* All done if we have committed all foreign transactions in one-phase */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	Assert(foreign_twophase_commit != FOREIGN_TWOPHASE_COMMIT_DISABLED);
+
+	/*
+	 * We now have only servers in the list which is capable of two-phase
+	 * commit. If the list has only one server and we didn't modify the local
+	 * data, we h can commit it in one-phase.
+	 */
+	if (list_length(FdwXactParticipants) == 1 && !local_modified)
+	{
+		/* Commit the foreign transaction in one-phase */
+		FdwXactOnePhaseEndForeignTransaction(linitial(FdwXactParticipants),
+											 true);
+
+		/* All foreign transaction must be committed */
+		list_free(FdwXactParticipants);
+		return;
+	}
+
+	/*
+	 * Finally, prepare foreign transactions. Note that we keep
+	 * FdwXactParticipants until the end of transaction.
+	 */
+	FdwXactPrepareForeignTransactions();
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here. If any error occurs, we rollback
+ * non-prepared foreign transactions and leave others to the resolver.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell   *lc;
+	TransactionId xid;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepared foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepared foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	xid = GetTopTransactionId();
+
+	/* Loop over the foreign connections */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		FdwXactRslvState *state;
+		FdwXact		fdwxact;
+
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, xid);
+
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the
+		 * FDWXACT_STATUS_PREPARING status. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before
+		 * persisting the information to the disk and crash in-between these
+		 * two steps, we will forget that we prepared the transaction on the
+		 * foreign server and will not be able to resolve it after the crash.
+		 * Hence persist first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(xid, fdw_part);
+
+		state = create_fdwxact_state();
+		state->server = fdw_part->server;
+		state->usermapping = fdw_part->usermapping;
+		state->fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+
+		/* Update the status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		Assert(fdwxact->status == FDWXACT_STATUS_INITIAL);
+		fdwxact->status = FDWXACT_STATUS_PREPARING;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 *
+		 * During abort processing, we might try to resolve a never-prepared
+		 * transaction, and get an error. This is fine as long as the FDW
+		 * provides us unique prepared transaction identifiers.
+		 */
+		PG_TRY();
+		{
+			fdw_part->prepare_foreign_xact_fn(state);
+		}
+		PG_CATCH();
+		{
+			/* failed, back to the initial state */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			fdwxact->status = FDWXACT_STATUS_INITIAL;
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		/* succeeded, update status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * One-phase commit or rollback the given foreign transaction participant.
+ */
+static void
+FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+									 bool for_commit)
+{
+	FdwXactRslvState *state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state = create_fdwxact_state();
+	state->server = fdw_part->server;
+	state->usermapping = fdw_part->usermapping;
+	state->flags = FDWXACT_FLAG_ONEPHASE;
+
+	/*
+	 * Commit or rollback foreign transaction in one-phase. Since we didn't
+	 * insert FdwXact entry for this transaction we don't need to care
+	 * failures. On failure we change to rollback.
+	 */
+	if (for_commit)
+		fdw_part->commit_foreign_xact_fn(state);
+	else
+		fdw_part->rollback_foreign_xact_fn(state);
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact		fdwxact;
+	FdwXactOnDiskData *fdwxact_file_data;
+	MemoryContext old_context;
+	int			data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							 fdw_part->usermapping->userid,
+							 fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_INITIAL;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			   Oid umid, char *fdwxact_id)
+{
+	int			i;
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("Duplicate entry with transaction id %u, serverid %u, userid %u exists.",
+									  xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->dbid = dbid;
+	fdwxact->local_xid = xid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	fdwxact->indoubt = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int			i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (FdwXactIsBeingResolved(fdwxact))
+		elog(ERROR, "cannot remove fdwxact entry that is being resolved");
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("Failed to find entry for xid %u, foreign server %u, and user %u.",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the WAL
+		 * record is inserted could complete without fsync'ing our state file.
+		 * (This is essentially the same kind of race condition as the
+		 * COMMIT-to-clog-write case that RecordTransactionCommit uses
+		 * delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and remove
+		 * the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/* Now we can mark ourselves as out of the commit critical section */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true and set ForeignTwophaseCommitIsRequired to true if the current
+ * transaction modifies data on two or more servers in FdwXactParticipants and
+ * local server itself. Also set *local_modified to true if the transaction modified
+ * the local data.
+ */
+static bool
+is_foreign_twophase_commit_required(bool *local_modified)
+{
+	ListCell   *lc;
+	int			nserverswritten = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+	{
+		++nserverswritten;
+		if (local_modified)
+			*local_modified = true;
+	}
+
+	/*
+	 * Atomic commit is required if we modified data on two or more
+	 * participants.
+	 */
+	if (nserverswritten <= 1)
+		return false;
+
+	ForeignTwophaseCommitIsRequired = true;
+	return true;
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int			i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell   *cell;
+	int			n_lefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact		fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		if (!fdw_part->fdwxact)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in
+		 * FdwXactParticipants could be used by other backend before we forget
+		 * in case where the resolver process removes the FdwXact entry and
+		 * other backend reuses it before we forget. So we need to check if
+		 * the entries are still associated with the transaction.
+		 */
+		SpinLockAcquire(&fdwxact->mutex);
+		if (fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+			fdwxact->indoubt = true;
+			n_lefts++;
+		}
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (n_lefts > 0)
+	{
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", n_lefts);
+		FdwXactComputeRequiredXmin();
+	}
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * Wait for the foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char	   *new_status = NULL;
+	const char *old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsForeignTwophaseCommitRequested())
+		return;
+
+	/*
+	 * Also exit if the transaction itself has no foreign transaction
+	 * participants.
+	 */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction resolution.
+	 */
+	if (update_process_title)
+	{
+		int			len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status, false);
+		new_status[len] = '\0'; /* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The
+		 * latter would lead the client to believe that the distributed
+		 * transaction aborted, which is not true: it's already committed
+		 * locally. The former is no good either: the client has requested
+		 * committing a distributed transaction, and is entitled to assume
+		 * that a acknowledged commit is also commit on all foreign servers,
+		 * which might not be true. So in this case we issue a WARNING (which
+		 * some clients may be able to interpret) and shut off further output.
+		 * We do NOT reset PorcDiePending, so that the process will die after
+		 * the commit is cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve
+		 * them later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status, false);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC	   *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC	   *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC	   *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lc;
+
+	if (!is_commit)
+	{
+		foreach(lc, FdwXactParticipants)
+		{
+			FdwXactParticipant *fdw_part = lfirst(lc);
+
+			/*
+			 * If the foreign transaction has FdwXact entry we might have
+			 * prepared it. Skip already-prepared foreign transaction because
+			 * it has closed its transaction. But we are not sure that foreign
+			 * transaction with status == FDWXACT_STATUS_PREPARING has been
+			 * prepared or not. So we call the rollback API to close its
+			 * transaction for safety. The prepared foreign transaction that
+			 * we might have will be resolved by the foreign transaction
+			 * resolver.
+			 */
+			if (fdw_part->fdwxact)
+			{
+				bool		is_prepared;
+
+				LWLockAcquire(FdwXactLock, LW_SHARED);
+				is_prepared = fdw_part->fdwxact &&
+					fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED;
+				LWLockRelease(FdwXactLock);
+
+				if (is_prepared)
+					continue;
+			}
+
+			/* One-phase rollback foreign transaction */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, false);
+		}
+	}
+
+	/*
+	 * In commit cases, we have already prepared foreign transactions during
+	 * pre-commit phase. And these prepared transactions will be resolved by
+	 * the resolver process.
+	 */
+
+	ForgetAllFdwXactParticipants();
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Prepare foreign transactions.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (is_foreign_twophase_commit_required(NULL) &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p)
+{
+	PGPROC	   *proc;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+			break;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+	{
+		*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+		*waitXid_p = proc->fdwXactWaitXid;
+	}
+	else
+	{
+		*nextResolutionTs_p = -1;
+		*waitXid_p = InvalidTransactionId;
+	}
+
+	LWLockRelease(FdwXactResolutionLock);
+
+	return proc;
+}
+
+/*
+ * Get one FdwXact entry to resolve. This function intended to be used when
+ * a resolver process get FdwXact entries to resolve. So we search entries
+ * while not including in-doubt transactions and in-progress transactions.
+ */
+static FdwXact
+get_fdwxact_to_resolve(Oid dbid, TransactionId xid)
+{
+	List	   *fdwxacts = NIL;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Don't include both in-doubt transactions and in-progress transactions */
+	fdwxacts = get_fdwxacts(dbid, xid, InvalidOid, InvalidOid,
+							false, false, false);
+
+	return fdwxacts == NIL ? NULL : (FdwXact) linitial(fdwxacts);
+}
+
+/*
+ * Resolve one distributed transaction on the given database . The target
+ * distributed transaction is fetched from the waiting queue and its transaction
+ * participants are fetched from the global array.
+ *
+ * Release the waiter and return true after we resolved the all of the foreign
+ * transaction participants. On failure, we re-enqueue the waiting backend after
+ * incremented the next resolution time.
+ */
+void
+FdwXactResolveTransaction(Oid dbid, TransactionId xid, PGPROC *waiter)
+{
+	FdwXact		fdwxact;
+
+	Assert(TransactionIdIsValid(xid));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	while ((fdwxact = get_fdwxact_to_resolve(MyDatabaseId, xid)) != NULL)
+	{
+		FdwXactRslvState *state;
+		ForeignServer *server;
+		UserMapping *usermapping;
+
+		CHECK_FOR_INTERRUPTS();
+
+		server = GetForeignServer(fdwxact->serverid);
+		usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+
+		state = create_fdwxact_state();
+		SpinLockAcquire(&fdwxact->mutex);
+		state->server = server;
+		state->usermapping = usermapping;
+		state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+		SpinLockRelease(&fdwxact->mutex);
+
+		FdwXactDetermineTransactionFate(fdwxact, false);
+
+		/* Do not hold during foreign transaction resolution */
+		LWLockRelease(FdwXactLock);
+
+		PG_TRY();
+		{
+			/*
+			 * Resolve the foreign transaction. When committing or aborting
+			 * prepared foreign transactions the previous status is always
+			 * FDWXACT_STATUS_PREPARED.
+			 */
+			FdwXactResolveForeignTransaction(fdwxact, state,
+											 FDWXACT_STATUS_PREPARED);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDWXACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				waiter->fdwXactNextResolutionTs =
+					TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+												foreign_xact_resolution_retry_interval);
+				FdwXactQueueInsert(waiter);
+			}
+			LWLockRelease(FdwXactResolutionLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved one foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter could
+	 * already be detached if user cancelled to wait before resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/*
+		 * Wake up the waiter only when we have set state and removed from
+		 * queue
+		 */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Determine whether the given foreign transaction should be committed or
+ * rolled back according to the result of the local transaction. This function
+ * changes fdwxact->status so the caller must hold FdwXactLock in exclusive
+ * mode or passing need_lock with true.
+ */
+static void
+FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock)
+{
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * The being resolved transaction must be either that has been cancelled
+	 * and marked as in-doubt or that has been prepared.
+	 */
+	Assert(fdwxact->indoubt ||
+		   fdwxact->status == FDWXACT_STATUS_PREPARED);
+
+	/*
+	 * If the local transaction is already committed, commit prepared foreign
+	 * transaction.
+	 */
+	if (TransactionIdDidCommit(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_COMMITTING;
+
+	/*
+	 * If the local transaction is already aborted, abort prepared foreign
+	 * transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+
+
+	/*
+	 * The local transaction is not in progress but the foreign transaction is
+	 * not prepared on the foreign server. This can happen when transaction
+	 * failed after registered this entry but before actual preparing on the
+	 * foreign server. So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is about
+	 * to be committed or aborted. This should not happen except for one case
+	 * where the local transaction is prepared and this foreign transaction is
+	 * being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid),
+				 errhint("The local transaction with xid %u might be prepared",
+						 fdwxact->local_xid)));
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * callback function. The 'state' is passed to the callback function. The fate of
+ * foreign transaction must be determined. If foreign transaction is resolved
+ * successfully, remove the FdwXact entry from the shared memory and also
+ * remove the corresponding on-disk file. If failed, the status of FdwXact
+ * entry changes to 'fallback_status' before erroring out.
+ */
+static void
+FdwXactResolveForeignTransaction(FdwXact fdwxact, FdwXactRslvState *state,
+								 FdwXactStatus fallback_status)
+{
+	ForeignServer *server;
+	ForeignDataWrapper *fdw;
+	FdwRoutine *fdw_routine;
+	bool		is_commit;
+
+	Assert(state != NULL);
+	Assert(state->server && state->usermapping && state->fdwxact_id);
+	Assert(fdwxact != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+		elog(ERROR, "cannot resolve foreign transaction whose fate is not determined");
+
+	is_commit = fdwxact->status == FDWXACT_STATUS_COMMITTING;
+	LWLockRelease(FdwXactLock);
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	PG_TRY();
+	{
+		if (is_commit)
+			fdw_routine->CommitForeignTransaction(state);
+		else
+			fdw_routine->RollbackForeignTransaction(state);
+	}
+	PG_CATCH();
+	{
+		/* Back to the fallback status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = fallback_status;
+		LWLockRelease(FdwXactLock);
+
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	elog(DEBUG1, "successfully %s the foreign transaction with xid %u db %u server %u user %u",
+		 is_commit ? "committed" : "rolled back",
+		 fdwxact->local_xid, fdwxact->dbid, fdwxact->serverid,
+		 fdwxact->userid);
+
+	fdwxact->status = FDWXACT_STATUS_RESOLVED;
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Return palloc'd and initialized FdwXactRslvState.
+ */
+static FdwXactRslvState *
+create_fdwxact_state(void)
+{
+	FdwXactRslvState *state;
+
+	state = palloc(sizeof(FdwXactRslvState));
+	state->server = NULL;
+	state->usermapping = NULL;
+	state->fdwxact_id = NULL;
+	state->flags = 0;
+
+	return state;
+}
+
+/*
+ * Return at least one FdwXact entry that matches to given argument,
+ * otherwise return NULL. All arguments must be valid values so that it can
+ * search exactly one (or none) entry. Note that this function intended to be
+ * used for modifying the returned FdwXact entry, so the caller must hold
+ * FdwXactLock in exclusive mode and it doesn't include the in-progress
+ * FdwXact entries.
+ */
+static FdwXact
+get_one_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	   *fdwxact_list;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	/* Include in-doubt transactions but don't include in-progress ones */
+	fdwxact_list = get_fdwxacts(dbid, xid, serverid, userid,
+								true, false, false);
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdwxact_list) <= 1);
+
+	/* Could not find entry */
+	if (fdwxact_list == NIL)
+		return NULL;
+
+	return (FdwXact) linitial(fdwxact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+FdwXactExists(Oid dbid, Oid serverid, Oid userid)
+{
+	List	   *fdwxact_list;
+	bool		ret = false;
+
+	/* Find entries from all FdwXact entries */
+	fdwxact_list = get_fdwxacts(dbid, InvalidTransactionId, serverid,
+								userid, true, true, true);
+
+	if (fdwxact_list != NIL)
+		ret = true;
+
+	list_free(fdwxact_list);
+	return ret;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_foreign_xacts, and the number of entries to num_p.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't
+ * want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdwxacts(int *num_p)
+{
+	List	   *all_fdwxacts;
+	ListCell   *lc;
+	FdwXact		fdwxacts;
+	int			num_fdwxacts = 0;
+
+	Assert(num_p != NULL);
+
+	/* Get all entries */
+	all_fdwxacts = get_fdwxacts(InvalidOid, InvalidTransactionId,
+								InvalidOid, InvalidOid, true,
+								true, true);
+
+	if (all_fdwxacts == NIL)
+	{
+		*num_p = 0;
+		return NULL;
+	}
+
+	fdwxacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdwxacts));
+	*num_p = list_length(all_fdwxacts);
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdwxacts)
+	{
+		FdwXact		fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdwxacts + num_fdwxacts, fx,
+			   sizeof(FdwXactData));
+		num_fdwxacts++;
+	}
+
+	list_free(all_fdwxacts);
+
+	return fdwxacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respectively.
+ */
+static List *
+get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			 bool include_indoubt, bool include_in_progress, bool need_lock)
+{
+	int			i;
+	List	   *fdwxact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* include in-doubt transaction? */
+		if (!include_indoubt && fdwxact->indoubt)
+			continue;
+
+		/* include in-progress transaction? */
+		if (!include_in_progress && FdwXactIsBeingResolved(fdwxact))
+			continue;
+
+		/* Append it if matched */
+		fdwxact_list = lappend(fdwxact_list, fdwxact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdwxact_list;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record in
+		 * FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transaction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	   *id;
+	int			id_len = 0;
+
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char		buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		/*
+		 * FDW doesn't provide the callback function, generate an unique
+		 * identifier.
+		 */
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+				 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifier \"%s\" is too long",
+						id),
+				 errdetail("Foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint's redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;					/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence of long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked invalid,
+	 * because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+				(errmsg_plural("%u foreign transaction state file was written "
+							   "for long-running prepared transactions",
+							   "%u foreign transaction state files were written "
+							   "for long-running prepared transactions",
+							   serialized_fdwxacts,
+							   serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
+									&read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+				 errdetail("Failed while allocating an XLog reading processor.")));
+
+	record = XLogReadRecord(xlogreader, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not read foreign transaction state from xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not recreate foreign transaction state file \"%s\": %m",
+						path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId origNextXid =
+	XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	   *buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open FDW transaction state file \"%s\": %m",
+						path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+	int			i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char	   *buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The status of
+	 * the transaction is set as preparing, since we do not know the exact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction which prepared this foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							 fdwxact_data->serverid, fdwxact_data->userid,
+							 fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know the xact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;		/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdwxact(dbid, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		return;
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int			i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(int *newval, void **extra, GucSource source)
+{
+	ForeignTwophaseCommitLevel newForeignTwophaseCommitLevel = *newval;
+
+	/* Parameter check */
+	if (newForeignTwophaseCommitLevel > FOREIGN_TWOPHASE_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+} WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	7
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdwxacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdwxacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(PG_PREPARED_FDWXACTS_COLS);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "indoubt",
+						   BOOLOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 7, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdwxacts = get_all_fdwxacts(&num_fdwxacts);
+		status->num_xacts = num_fdwxacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdwxact = &status->fdwxacts[status->cur_xact++];
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdwxact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdwxact->dbid);
+		values[1] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[2] = ObjectIdGetDatum(fdwxact->serverid);
+		values[3] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (fdwxact->status)
+		{
+			case FDWXACT_STATUS_INITIAL:
+				xact_status = "initial";
+				break;
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			case FDWXACT_STATUS_RESOLVED:
+				xact_status = "resolved";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		values[5] = BoolGetDatum(fdwxact->indoubt);
+		values[6] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			serverid = PG_GETARG_OID(1);
+	Oid			userid = PG_GETARG_OID(2);
+	ForeignServer *server;
+	UserMapping *usermapping;
+	FdwXact		fdwxact;
+	FdwXactRslvState *state;
+	FdwXactStatus prev_status;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	server = GetForeignServer(serverid);
+	usermapping = GetUserMapping(userid, serverid);
+	state = create_fdwxact_state();
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+	{
+		LWLockRelease(FdwXactLock);
+		PG_RETURN_BOOL(false);
+	}
+
+	state->server = server;
+	state->usermapping = usermapping;
+	state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+
+	SpinLockAcquire(&fdwxact->mutex);
+	prev_status = fdwxact->status;
+	SpinLockRelease(&fdwxact->mutex);
+
+	FdwXactDetermineTransactionFate(fdwxact, false);
+
+	LWLockRelease(FdwXactLock);
+
+	FdwXactResolveForeignTransaction(fdwxact, state, prev_status);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			serverid = PG_GETARG_OID(1);
+	Oid			userid = PG_GETARG_OID(2);
+	FdwXact		fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000000..e293d13562
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,641 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to retry resolution.
+ */
+void
+FdwXactLauncherRequestToLaunchForRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool		found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int			slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			resolver->last_resolved_time = 0;
+			resolver->latch = NULL;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz now;
+		long		wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int			rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a
+		 * foreign_xact_resolution_retry_interval but always starts when the
+		 * backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool		launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested but not
+			 * running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we should
+			 * retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool		found = false;
+	int			i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the same
+	 * database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find
+		 * FdwXact entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int			unused_slot = -1;
+	int			i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign transaction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to
+	 * wait until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	   *resolver_dbs;	/* DBs resolver's running on */
+	HTAB	   *fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL		ctl;
+	HASH_SEQ_STATUS status;
+	Oid		   *entry;
+	bool		launched;
+	int			i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->indoubt)
+			continue;
+
+		hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+		return false;
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool		found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	return launched;
+}
+
+/* Register a background worker running the foreign transaction launcher */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid			dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int			i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign transaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int			rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		/* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+					   10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Returns activity of all foreign transaction resolvers.
+ */
+Datum
+pg_stat_get_foreign_xact(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int			i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t		pid;
+		Oid			dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000000..4843aeacc9
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,343 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int			foreign_xact_resolution_retry_interval;
+int			foreign_xact_resolver_timeout = 60 * 1000;
+bool		foreign_xact_resolve_indoubt_xacts;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	MyFdwXactResolver->last_resolved_time = 0;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int			slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		PGPROC	   *waiter = NULL;
+		TransactionId waitXid = InvalidTransactionId;
+		TimestampTz resolutionTs = -1;
+		int			rc;
+		TimestampTz now;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or got the waiter
+		 * that has future resolution time.
+		 */
+		while ((waiter = FdwXactGetWaiter(&resolutionTs, &waitXid)) != NULL)
+		{
+			CHECK_FOR_INTERRUPTS();
+			Assert(TransactionIdIsValid(waitXid));
+
+			if (resolutionTs > now)
+				break;
+
+			elog(DEBUG2, "resolver got one waiter with xid %u", waitXid);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveTransaction(MyDatabaseId, waitXid, waiter);
+			CommitTransactionCommand();
+
+			/* Update my stats */
+			SpinLockAcquire(&(MyFdwXactResolver->mutex));
+			MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+			SpinLockRelease(&(MyFdwXactResolver->mutex));
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz last_resolved_time;
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	last_resolved_time = MyFdwXactResolver->last_resolved_time;
+	timeout = TimestampTzPlusMilliseconds(last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * checked FdwXactWaiterExists.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long		sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index f88d72fd86..982c1a36cc 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -13,6 +13,7 @@ OBJS = \
 	clogdesc.o \
 	committsdesc.o \
 	dbasedesc.o \
+	fdwxactdesc.o \
 	genericdesc.o \
 	gindesc.o \
 	gistdesc.o \
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000000..ca761763e5
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxactdesc.c
+ *		PostgreSQL global transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdwxactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdwxact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		FdwXactOnDiskData *fdwxact_insert = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_insert->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_insert->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_insert->dbid);
+		appendStringInfo(buf, " local xid: %u,", fdwxact_insert->local_xid);
+		appendStringInfo(buf, " id: %s", fdwxact_insert->fdwxact_id);
+	}
+	else
+	{
+		xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_remove->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_remove->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_remove->dbid);
+		appendStringInfo(buf, " local xid: %u", fdwxact_remove->xid);
+	}
+
+}
+
+const char *
+fdwxact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDWXACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDWXACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 1cd97852e8..ea045174e0 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 58091f6b52..200cf9d067 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -10,6 +10,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 5adf956f41..e8e6a5e2b5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -850,6 +851,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 	return result;
 }
 
+/*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
 /*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
@@ -2263,6 +2293,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2322,6 +2358,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index e3c60f23cd..405271387d 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1218,6 +1219,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1226,6 +1228,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1264,12 +1267,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1427,6 +1431,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2086,6 +2098,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2246,6 +2261,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2333,6 +2349,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2527,6 +2545,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2732,6 +2751,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 3813eadfb4..f69e572b1c 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
@@ -5243,6 +5244,7 @@ BootStrapXLOG(void)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6182,6 +6184,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_wal_senders",
 									 max_wal_senders,
 									 ControlFile->max_wal_senders);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
@@ -6723,14 +6728,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -6922,7 +6928,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7431,6 +7440,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7761,6 +7771,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9037,6 +9050,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9470,8 +9484,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9489,6 +9505,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9505,6 +9522,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9710,6 +9728,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -9909,6 +9928,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index f681aafcf9..980ddbad0a 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -333,6 +333,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
@@ -826,6 +829,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_foreign_xact AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_foreign_xact() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index e79ede4cb8..2293d4cbfc 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2858,8 +2858,14 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
+	}
 
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index f197869752..6206265424 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1101,6 +1103,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (FdwXactExists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1419,6 +1433,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
+	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (FdwXactExists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
 	/*
 	 * Do the deletion
 	 */
@@ -1572,6 +1595,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index c13b1d3501..1dc61fbdea 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/table.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
@@ -937,7 +938,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 513471ab9b..29f376e48c 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,9 +226,31 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
+	}
 
 	return scanstate;
 }
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 870a7428f1..d82d32ecb4 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -2411,6 +2413,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index 61e48ca3f8..8f411c0559 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMappingOid - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,18 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction && !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction && routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper must support both commit and rollback routines or neither");
+
+	if (routine->PrepareForeignTransaction &&
+		!routine->CommitForeignTransaction &&
+		!routine->RollbackForeignTransaction)
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 75fc0d5d33..c8b38142a5 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -12,6 +12,8 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -128,6 +130,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 59dc4f31ab..9dce03a6e4 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3645,6 +3645,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3848,6 +3854,11 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDWXACT:
+			event_name = "FdwXact";
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -4066,6 +4077,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index b3986bee75..897863a79c 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -909,6 +911,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -984,12 +990,13 @@ PostmasterMain(int argc, char *argv[])
 #endif
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index 5e1dc8a651..c77ca40e1c 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -151,6 +151,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 427b0d59cd..55609eed81 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(void)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(void)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 4a5b26c23d..5d54acaaf5 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -93,6 +93,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -248,6 +250,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1312,6 +1315,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1377,6 +1381,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1426,6 +1431,15 @@ GetOldestXmin(Relation rel, int flags)
 		NormalTransactionIdPrecedes(replication_slot_xmin, result))
 		result = replication_slot_xmin;
 
+	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
 	/*
 	 * After locks have been released and vacuum_defer_cleanup_age has been
 	 * applied, check whether we need to back up further to make logical
@@ -3128,6 +3142,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations of the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index db47843229..adb276370c 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -49,3 +49,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 CLogTruncationLock					44
+FdwXactLock							45
+FdwXactResolverLock					46
+FdwXactResolutionLock				47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index eb321f72ea..7c7edeeaaf 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -421,6 +422,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* Initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -822,6 +827,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 0a6f80963b..9b40134aa4 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3030,6 +3032,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 8228e1f390..9304ebbd76 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -424,6 +425,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 	{NULL, 0, false}
 };
 
+/*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
 /*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
@@ -760,6 +780,12 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
+	/* FDWXACT_RESOLVER */
+	gettext_noop("Foreign Transaction Management / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2448,6 +2474,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FDWXACT_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FDWXACT_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4491,6 +4563,16 @@ static struct config_enum ConfigureNamesEnum[] =
 		NULL, assign_synchronous_commit, NULL
 	},
 
+	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
 	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index e1048c0047..859f32eb95 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -342,6 +344,20 @@
 #max_sync_workers_per_subscription = 2	# taken from max_logical_replication_workers
 
 
+#------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#foreign_twophase_commit = off
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
 #------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index a0b0458108..8701c5f005 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 5302973379..c448ce7373 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -208,6 +208,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index 19e21ab491..9ae3bfe4dd 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -301,6 +301,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index f9cfeae264..a5f2aa1a09 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -710,6 +710,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -914,6 +915,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/fdwxactdesc.c b/src/bin/pg_waldump/fdwxactdesc.c
new file mode 120000
index 0000000000..ce8c21880c
--- /dev/null
+++ b/src/bin/pg_waldump/fdwxactdesc.c
@@ -0,0 +1 @@
+../../../src/backend/access/rmgrdesc/fdwxactdesc.c
\ No newline at end of file
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca4b1..b616cea347 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000000..dd8433f42c
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,167 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL global transaction manager
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDWXACT_H
+#define FDWXACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* fdwXactState */
+#define	FDWXACT_NOT_WAITING		0
+#define	FDWXACT_WAITING			1
+#define	FDWXACT_WAIT_COMPLETE	2
+
+/* Flag passed to FDW transaction management APIs */
+#define FDWXACT_FLAG_ONEPHASE		0x01	/* transaction can commit/rollback
+											 * without preparation */
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER, /* use twophase commit where available */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support
+										 * twophase commit */
+} ForeignTwophaseCommitLevel;
+
+/* Enum to track the status of foreign transaction */
+typedef enum
+{
+	FDWXACT_STATUS_INVALID = 0,
+	FDWXACT_STATUS_INITIAL,
+	FDWXACT_STATUS_PREPARING,	/* foreign transaction is being prepared */
+	FDWXACT_STATUS_PREPARED,	/* foreign transaction is prepared */
+	FDWXACT_STATUS_COMMITTING,	/* foreign prepared transaction is being
+								 * committed */
+	FDWXACT_STATUS_ABORTING,	/* foreign prepared transaction is being
+								 * aborted */
+	FDWXACT_STATUS_RESOLVED
+} FdwXactStatus;
+
+typedef struct FdwXactData *FdwXact;
+
+/*
+ * Shared memory state of a single foreign transaction.
+ */
+typedef struct FdwXactData
+{
+	FdwXact		fdwxact_free_next;	/* Next free FdwXact entry */
+
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	TransactionId local_xid;	/* XID of local transaction */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	bool		indoubt;		/* Is an in-doubt transaction? */
+	slock_t		mutex;			/* Protect the above fields */
+
+	/* The status of the foreign transaction, protected by FdwXactLock */
+	FdwXactStatus status;
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;	/* XLOG offset of inserting this entry
+									 * start */
+	XLogRecPtr	insert_end_lsn; /* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to
+								 * file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* prepared transaction
+												 * identifier */
+}			FdwXactData;
+
+/*
+ * Shared memory layout for maintaining foreign prepared transaction entries.
+ * Adding or removing FdwXact entry needs to hold FdwXactLock in exclusive mode,
+ * and iterating fdwXacts needs that in shared mode.
+ */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		free_fdwxacts;
+
+	/* Number of valid foreign transaction entries */
+	int			num_fdwxacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdwxacts[FLEXIBLE_ARRAY_MEMBER];	/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* State data for foreign transaction resolution, passed to FDW callbacks */
+typedef struct FdwXactRslvState
+{
+	/* Foreign transaction information */
+	char	   *fdwxact_id;
+
+	ForeignServer *server;
+	UserMapping *usermapping;
+
+	int			flags;			/* OR of FDWXACT_FLAG_xx flags */
+} FdwXactRslvState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+/* Function declarations */
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void RegisterFdwXactByRelId(Oid relid, bool modified);
+extern void RegisterFdwXactByServerId(Oid serverid, bool modified);
+extern void PreCommit_FdwXacts(void);
+extern void FdwXactResolveTransaction(Oid dbid, TransactionId xid, PGPROC *waiter);
+extern bool FdwXactResolveInDoubtTransactions(Oid dbid);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern PGPROC *FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p);
+extern bool FdwXactWaiterExists(Oid dbid);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void RecoverFdwXacts(void);
+extern bool FdwXactExists(Oid dboid, Oid serverid, Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+extern void FdwXactMarkForeignServerAccessed(Oid relid, bool modified);
+extern bool check_foreign_twophase_commit(int *newval, void **extra,
+										  GucSource source);
+
+#endif							/* FDWXACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000000..c3ed1ecfaf
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,29 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherRequestToLaunch(void);
+extern void FdwXactLauncherRequestToLaunchForRetry(void);
+extern void FdwXactLaunchOrWakeupResolver(void);
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+
+
+#endif							/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000000..779848113c
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int	foreign_xact_resolver_timeout;
+
+#endif							/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000000..b4cec76eae
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,54 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDWXACT_INSERT	0x00
+#define XLOG_FDWXACT_REMOVE	0x10
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDWXACT_ID_MAX_LEN 200
+
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdwxact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+	bool		force;
+} xl_fdwxact_remove;
+
+extern void fdwxact_redo(XLogReaderState *record);
+extern void fdwxact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdwxact_identify(uint8 info);
+
+#endif							/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000000..80691b5c07
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,66 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/resolver_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t		pid;			/* this resolver's PID, or 0 if not active */
+	Oid			dbid;			/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool		in_use;
+
+	/* Stats */
+	TimestampTz last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t		mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	   *latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/* Foreign transaction resolution queue. Protected by FdwXactLock */
+	SHM_QUEUE	fdwxact_queue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch	   *launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+#define SizeOfFdwXactRslvCtlData \
+	(offsetof(FdwXactRslvCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif							/* RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index c88dccfb8d..254a663b4d 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Foreign Transactions", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 2ca71c3445..bd027a2861 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index a04fc70326..6f1f336e31 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -108,6 +108,13 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
+/*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 087918d41d..ed4d08b4af 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -232,6 +232,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index de5670e538..9884f5f8e7 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index eb3c1a88d1..49465a1df2 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5211,6 +5211,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '9705', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_foreign_xact', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_foreign_xact' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5924,6 +5931,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '9706', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o,o}',
+  proargnames => '{dbid,xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '9707', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '9708', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -6042,6 +6067,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '9709',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 95556dfb15..8d046cc4e4 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 5e0cf533fb..5596ee591c 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -69,6 +69,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 											   bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 														 bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index 3a65a51696..a59d7c1f60 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -776,6 +776,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -853,7 +855,9 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDWXACT,
+	WAIT_EVENT_FDWXACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -934,6 +938,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index d21780108b..35ffbbca93 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -16,6 +16,7 @@
 
 #include "access/clog.h"
 #include "access/xlogdefs.h"
+#include "datatype/timestamp.h"
 #include "lib/ilist.h"
 #include "storage/latch.h"
 #include "storage/lock.h"
@@ -152,6 +153,16 @@ struct PGPROC
 	int			syncRepState;	/* wait state for sync rep */
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
+	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
 	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index a5c7d0c064..0f73b64937 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -125,4 +127,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 454c2df487..6010dbcdee 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,9 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
+	FDWXACT_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 634f8256f7..0f3ff9742e 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1342,6 +1342,14 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.dbid,
+    f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(dbid, xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
@@ -1848,6 +1856,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_foreign_xact| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_foreign_xact() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_gssapi| SELECT s.pid,
     s.gss_auth AS gss_authenticated,
     s.gss_princ AS principal,
-- 
2.23.0

From 0af83b7a06d6374b3c2e72afc7e460fa12fc314e Mon Sep 17 00:00:00 2001
From: Muhammad Usama <m.usama@highgo.ca>
Date: Thu, 26 Mar 2020 21:41:29 +0500
Subject: [PATCH v19 5/5] Add regression tests for atomic commit.

Authors: Muhammad Usama, Masahiko Sawada, Ahutosh Bapat
---
 src/test/recovery/Makefile         |   2 +-
 src/test/recovery/t/016_fdwxact.pl | 175 +++++++++++++++++++++++++++++
 src/test/regress/pg_regress.c      |  13 ++-
 3 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 src/test/recovery/t/016_fdwxact.pl

diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index fa8e031526..d47d96975b 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/016_fdwxact.pl b/src/test/recovery/t/016_fdwxact.pl
new file mode 100644
index 0000000000..9af9bb81dc
--- /dev/null
+++ b/src/test/recovery/t/016_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index f6a5e1b9c7..9f62e750b5 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2320,9 +2320,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2337,7 +2340,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m %b[%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.21.1 (Apple Git-122.3)

From 92823fb0fe93eed7c91bf2628fcd3fed8694e69d Mon Sep 17 00:00:00 2001
From: Muhammad Usama <m.usama@highgo.ca>
Date: Thu, 26 Mar 2020 21:22:17 +0500
Subject: [PATCH v19 2/5] Support atomic commit among multiple foreign servers.

Authors: Muhammad Usama, Masahiko Sawada, Ahutosh Bapat
---
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  130 +
 src/backend/access/fdwxact/fdwxact.c          | 2888 +++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  641 ++++
 src/backend/access/fdwxact/resolver.c         |  343 ++
 src/backend/access/rmgrdesc/Makefile          |    1 +
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   58 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   42 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |   11 +
 src/backend/commands/copy.c                   |    6 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    8 +
 src/backend/executor/nodeForeignscan.c        |   24 +
 src/backend/executor/nodeModifyTable.c        |    6 +
 src/backend/foreign/foreign.c                 |   55 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   20 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   82 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/fdwxactdesc.c              |    1 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  167 +
 src/include/access/fdwxact_launcher.h         |   29 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   54 +
 src/include/access/resolver_internal.h        |   66 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   29 +
 src/include/foreign/fdwapi.h                  |   12 +
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    9 +-
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    3 +
 src/test/regress/expected/rules.out           |   13 +
 55 files changed, 4972 insertions(+), 18 deletions(-)
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100644 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 100644 src/bin/pg_waldump/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a8bb..49480dd039 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000000..0207a66fb4
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000000..c20570022c
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,130 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a consistent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+---------------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consists
+of the following four steps:
+
+1. Foreign Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant. The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+we record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE all foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared transaction on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step (commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itself to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rollbacked the resolver process wake up the waiter.
+
+
+Foreign Data Wrapper Callbacks for Transaction Management
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForeignTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for either PREPARE, COMMIT or
+ROLLBACK the transaction on the foreign server respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transaction in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions has an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_INITIAL
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_PREPARING, FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING
+before the foreign transaction is prepared, committed and aborted by FDW
+callback functions respectively(*1). And the status then changes to
+FDWXACT_STATUS_RESOLVED once the foreign transaction are resolved, and then
+the corresponding FdwXact entry is removed with WAL logging. If failed during
+processing foreign transaction (e.g., preparing, committing or aborting) the
+status changes back to the previous status. Therefore the status
+FDWXACT_STATUS_xxxING appear only during the foreign transaction is being
+processed by an FDW callback function.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*2). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                      INVALID                       |
+ +----------------------------------------------------+
+    |                      |                       |
+    |                      v                       |
+    |           +---------------------+            |
+    |           |       INITIAL       |            |
+    |           +---------------------+            |
+   (*2)                    |                      (*2)
+    |                      v                       |
+    |           +---------------------+            |
+    |           |    PREPARING(*1)    |            |
+    |           +---------------------+            |
+    |                      |                       |
+    v                      v                       v
+ +----------------------------------------------------+
+ |                      PREPARED                      |
+ +----------------------------------------------------+
+           |                               |
+           v                               v
+ +--------------------+          +--------------------+
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |
+ +--------------------+          +--------------------+
+           |                               |
+           v                               v
+ +----------------------------------------------------+
+ |                      RESOLVED                      |
+ +----------------------------------------------------+
+
+(*1) Status that appear only during being processed by FDW
+(*2) Paths for recovered FdwXact entries
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100644
index 0000000000..0990a4e3ed
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2888 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers atomically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit. The foreign servers are
+ * registered if FDW has both CommitForeignTransaction API and
+ * RollbackForeignTransaction API. Registered participant servers are identified
+ * by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * foreign server everywhere. And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask it to commit, we also tell it to notify us when
+ * it's done, so that we can wait interruptibly for it to finish, and so
+ * that we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request. We have one queue
+ * of which elements are ordered by the timestamp that they expect to be
+ * processed at. Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp that they expects to be processed.
+ * Similarly if failed to resolve them, it enqueues again with new timestamp
+ * (its timestamp + foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction). Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by
+ * pg_resovle_foreign_xact() function.
+ *
+ * Two-phase commit protocol is required if the transaction modified two or
+ * more servers including itself. In other case, all foreign transactions are
+ * committed or rolled back during pre-commit.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed by FDW, the corresponding
+ * FdwXact entry is update. In order to protect the entry from concurrent
+ * removing we need to hold a lock on the entry or a lock for entire global
+ * array. However, we don't want to hold the lock during FDW is processing the
+ * foreign transaction that may take a unpredictable time. To avoid this, the
+ * in-memory data of foreign transaction follows a locking model based on
+ * four linked concepts:
+ *
+ * * A foreign transaction's status variable is switched using the LWLock
+ *   FdwXactLock, which need to be hold in exclusive mode when updating the
+ *   status, while readers need to hold it in shared mode when looking at the
+ *   status.
+ * * A process who is going to update FdwXact entry cannot process foreign
+ *   transaction that is being resolved.
+ * * So setting the status to FDWXACT_STATUS_PREPARING,
+ *   FDWXACT_STATUS_COMMITTING or FDWXACT_STATUS_ABORTING, which makes foreign
+ *   transaction in-progress states, means to own the FdwXact entry, which
+ *   protect it from updating/removing by concurrent writers.
+ * * Individual fields are protected by mutex where only the backend owning
+ *   the foreign transaction is authorized to update the fields from its own
+ *   one.
+ *
+ * Therefore, before doing PREPARE, COMMIT PREPARED or ROLLBACK PREPARED a
+ * process who is going to call transaction callback functions needs to change
+ * the status to the corresponding status above while holding FdwXactLock in
+ * exclusive mode, and call callback function after releasing the lock.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk. FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon. We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts. If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to disk.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Atomic commit is enabled by configuration */
+#define IsForeignTwophaseCommitEnabled() \
+	(max_prepared_foreign_xacts > 0 && \
+	 max_foreign_xact_resolvers > 0)
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	(IsForeignTwophaseCommitEnabled() && \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED))
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define IsSeverCapableOfTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/* Check the FdwXact is begin resolved */
+#define FdwXactIsBeingResolved(fx) \
+	(((((FdwXact)(fx))->status) == FDWXACT_STATUS_PREPARING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_COMMITTING) || \
+	 ((((FdwXact)(fx))->status) == FDWXACT_STATUS_ABORTING))
+
+/*
+ * Structure to bundle the foreign transaction participant. This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry is
+	 * not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer *server;
+	UserMapping *usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char	   *fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool		modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function prepare_foreign_xact_fn;
+	CommitForeignTransaction_function commit_foreign_xact_fn;
+	RollbackForeignTransaction_function rollback_foreign_xact_fn;
+	GetPrepareId_function get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transaction participants for atomic commit. This list
+ * has only foreign servers that provides transaction management callbacks,
+ * that is CommitForeignTransaction and RollbackForeignTransaction.
+ */
+static List *FdwXactParticipants = NIL;
+
+/*
+ * True is the current transaction needs to be committed together with
+ * foreign servers.
+ */
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int			max_prepared_foreign_xacts = 0;
+int			max_foreign_xact_resolvers = 0;
+int			foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid,
+										 FdwXactParticipant *fdw_part);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+												 bool for_commit);
+static void FdwXactResolveForeignTransaction(FdwXact fdwxact,
+											 FdwXactRslvState *state,
+											 FdwXactStatus fallback_status);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid, void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static void FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock);
+static bool is_foreign_twophase_commit_required(bool *local_modified);
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static List *get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						  bool including_indoubts, bool include_in_progress,
+						  bool need_lock);
+static FdwXact get_all_fdwxacts(int *num_p);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXact get_fdwxact_to_resolve(Oid dbid, TransactionId xid);
+static FdwXactRslvState *create_fdwxact_state(void);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation	rel;
+	Oid			serverid;
+	Oid			userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction.
+ *
+ * The foreign transaction identified by given server id and user id.
+ * Registered foreign transactions are managed by the global transaction
+ * manager until the end of the transaction.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant *fdw_part;
+	ForeignServer *foreign_server;
+	UserMapping *user_mapping;
+	MemoryContext old_ctx;
+	FdwRoutine *routine;
+	ListCell   *lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Don't register foreign server if it doesn't provide both commit and
+	 * rollback transaction management callbacks and is modified.
+	 */
+	if (!routine->CommitForeignTransaction &&
+		!routine->RollbackForeignTransaction &&
+		modified)
+	{
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+		pfree(routine);
+		return;
+	}
+
+	/*
+	 * Remember we modified on the foreign server that is not capable of two-phase
+	 * commit.
+	 */
+	if (!routine->PrepareForeignTransaction &&
+		modified)
+		MyXactFlags |= XACT_FLAGS_FDWNOPREPARE;
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	foreign_server = GetForeignServer(serverid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact = NULL;
+	fdw_part->modified = modified;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&(fdwxacts[cnt].mutex));
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * When foreign twophase commit is enabled, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign server's FDWs to support two-phase commit protocol and ask them to
+ * prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and for 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	bool		need_twophase_commit;
+	bool		local_modified;
+	ListCell   *lc = NULL;
+	List       *non_twophase_participants = NIL;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false if
+	 * foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = is_foreign_twophase_commit_required(&local_modified);
+
+	/*
+	 * If foreign two phase commit is required then all foreign serves
+	 * must be be capable of doing two-phase commit
+	 */
+
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0 &&
+		need_twophase_commit)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot COMMIT a distributed transaction that has operated on a foreign server that doesn't support atomic commit")));
+
+	/* Attempt to commit foreign transactions in one-phase */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		bool		commit = false;
+		if (!need_twophase_commit)
+		{
+			/* Can commit in one-phase if two-phase commit is not required */
+			commit = true;
+		}
+		else if (!fdw_part->modified)
+		{
+			/*
+			 * Non-modified foreign transaction always can be committed in
+			 * one-phase regardless of two-phase commit support.
+			 */
+			commit = true;
+		}
+		else if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER &&
+				 !IsSeverCapableOfTwophaseCommit(fdw_part))
+		{
+			/*
+			 * In 'prefer' mode, transactions for the servers lacking the
+			 * capability of a two-phase commit can be committed using
+			 * single-phase, but instead of doing the commit right away,
+			 * we wait until all prepared transaction gets prepared successfully
+			 * on two-phase transaction capable servers.So just add this server
+			 * to the list and process that after preparing foreign transactions
+			 */
+			non_twophase_participants = lappend(non_twophase_participants, fdw_part);
+			FdwXactParticipants = foreach_delete_current(FdwXactParticipants, lc);
+		}
+
+		if (commit)
+		{
+			/* Commit the foreign transaction in one-phase */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, true);
+
+			/* Delete it from the participant list */
+			FdwXactParticipants = foreach_delete_current(FdwXactParticipants, lc);
+			pfree(fdw_part);
+		}
+	}
+
+	/* All done if we have already committed all foreign transactions */
+	if (FdwXactParticipants == NIL && non_twophase_participants == NIL)
+		return;
+
+	Assert(foreign_twophase_commit != FOREIGN_TWOPHASE_COMMIT_DISABLED);
+
+	/*
+	 * If we now have only one two-phase capable server left in the list
+	 * no server in non_twophase_participants list
+	 * we didn't modify the local
+	 * data, then we don't need the two phase commit.
+	 */
+	if (non_twophase_participants == NIL &&
+		list_length(FdwXactParticipants) == 1 &&
+		!local_modified)
+	{
+		/* Commit the foreign transaction in one-phase */
+		FdwXactOnePhaseEndForeignTransaction(linitial(FdwXactParticipants),
+											 true);
+
+		list_free_deep(FdwXactParticipants);
+		FdwXactParticipants = NIL;
+		return;
+	}
+
+	/*
+	 * Finally, prepare foreign transactions. Note that we keep
+	 * FdwXactParticipants until the end of transaction.
+	 */
+	if (FdwXactParticipants)
+	{
+		FdwXactPrepareForeignTransactions();
+
+		/*
+		 * set ForeignTwophaseCommitIsRequired, if we have prepared the
+		 * transactions on the foreign servers
+		 */
+		ForeignTwophaseCommitIsRequired = true;
+	}
+
+	if (non_twophase_participants == NIL)
+		return;
+	/*
+	 * commit the transactions on servers lacking two-phase capability.
+	 */
+	foreach(lc, non_twophase_participants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		/* Commit the foreign transaction in one-phase */
+		FdwXactOnePhaseEndForeignTransaction(fdw_part, true);
+	}
+	list_free_deep(non_twophase_participants);
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here. If any error occurs, we rollback
+ * non-prepared foreign transactions and leave others to the resolver.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell   *lc;
+	TransactionId xid;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Parameter check */
+	if (max_prepared_foreign_xacts == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepared foreign transactions are disabled"),
+				 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+	if (max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("prepared foreign transactions are disabled"),
+				 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+	xid = GetTopTransactionId();
+
+	/* Loop over the foreign connections */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		FdwXactRslvState *state;
+		FdwXact		fdwxact;
+
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, xid);
+
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the
+		 * FDWXACT_STATUS_PREPARING status. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before
+		 * persisting the information to the disk and crash in-between these
+		 * two steps, we will lost the prepared transaction on the foreign
+		 * server and will not be able to resolve it after the crash recovery.
+		 * Hence persist first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(xid, fdw_part);
+
+		state = create_fdwxact_state();
+		state->server = fdw_part->server;
+		state->usermapping = fdw_part->usermapping;
+		state->fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+
+		/* Update the status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		Assert(fdwxact->status == FDWXACT_STATUS_INITIAL);
+		fdwxact->status = FDWXACT_STATUS_PREPARING;
+		LWLockRelease(FdwXactLock);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 *
+		 * During abort processing, we might try to resolve a never-prepared
+		 * transaction, and get an error. This is fine as long as the FDW
+		 * provides us unique prepared transaction identifiers.
+		 */
+		PG_TRY();
+		{
+			fdw_part->prepare_foreign_xact_fn(state);
+		}
+		PG_CATCH();
+		{
+			/* failed, back to the initial state */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			fdwxact->status = FDWXACT_STATUS_INITIAL;
+			LWLockRelease(FdwXactLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		/* succeeded, update status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/*
+ * One-phase commit or rollback the given foreign transaction participant.
+ */
+static void
+FdwXactOnePhaseEndForeignTransaction(FdwXactParticipant *fdw_part,
+									 bool for_commit)
+{
+	FdwXactRslvState *state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state = create_fdwxact_state();
+	state->server = fdw_part->server;
+	state->usermapping = fdw_part->usermapping;
+	state->flags = FDWXACT_FLAG_ONEPHASE;
+
+	/*
+	 * Commit or rollback foreign transaction in one-phase. Since we didn't
+	 * insert FdwXact entry for this transaction we don't need to care
+	 * failures. On failure we change to rollback.
+	 */
+	if (for_commit)
+		fdw_part->commit_foreign_xact_fn(state);
+	else
+		fdw_part->rollback_foreign_xact_fn(state);
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact		fdwxact;
+	FdwXactOnDiskData *fdwxact_file_data;
+	MemoryContext old_context;
+	int			data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							 fdw_part->usermapping->userid,
+							 fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_INITIAL;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyPgXact->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyPgXact->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			   Oid umid, char *fdwxact_id)
+{
+	int			i;
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("Duplicate entry with transaction id %u, serverid %u, userid %u exists.",
+									  xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->dbid = dbid;
+	fdwxact->local_xid = xid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	fdwxact->indoubt = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int			i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (FdwXactIsBeingResolved(fdwxact))
+		elog(ERROR, "cannot remove fdwxact entry that is being resolved");
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("Failed to find entry for xid %u, foreign server %u, and user %u.",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the WAL
+		 * record is inserted could complete without fsync'ing our state file.
+		 * (This is essentially the same kind of race condition as the
+		 * COMMIT-to-clog-write case that RecordTransactionCommit uses
+		 * delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyPgXact->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and remove
+		 * the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/* Now we can mark ourselves as out of the commit critical section */
+		MyPgXact->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return true if the current transaction modifies data on two or more servers
+ * in FdwXactParticipants and local server itself.
+ * Also set *local_modified to true if the transaction modified the local data.
+ */
+static bool
+is_foreign_twophase_commit_required(bool *local_modified)
+{
+	ListCell   *lc;
+	int			nserverswritten = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->modified)
+			nserverswritten++;
+	}
+
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+	{
+		++nserverswritten;
+		if (local_modified)
+			*local_modified = true;
+	}
+
+	/*
+	 * Atomic commit is required if we modified data on two or more
+	 * participants.
+	 */
+	if (nserverswritten <= 1)
+		return false;
+
+	return true;
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	int			i;
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell   *cell;
+	int			n_lefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact		fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		if (!fdw_part->fdwxact)
+			continue;
+
+		/*
+		 * There is a race condition; the FdwXact entries in
+		 * FdwXactParticipants could be used by other backend before we forget
+		 * in case where the resolver process removes the FdwXact entry and
+		 * other backend reuses it before we forget. So we need to check if
+		 * the entries are still associated with the transaction.
+		 */
+		SpinLockAcquire(&fdwxact->mutex);
+		if (fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+			fdwxact->indoubt = true;
+			n_lefts++;
+		}
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (n_lefts > 0)
+	{
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", n_lefts);
+		FdwXactComputeRequiredXmin();
+	}
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * Wait for the foreign transaction to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitToBeResolved(TransactionId wait_xid, bool is_commit)
+{
+	char	   *new_status = NULL;
+	const char *old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/* Quick exit if atomic commit is not requested */
+	if (!IsForeignTwophaseCommitRequested())
+		return;
+
+	/*
+	 * exit if the transaction itself has no foreign transaction
+	 * participants
+	 */
+	if (FdwXactParticipants == NIL)
+	{
+		/*
+		 * If we are here because of COMMIT/ROLLBACK PREPARED then the
+		 * FdwXactParticipants list would be empty. So we need to
+		 * see if there are any foreign prepared transactions exists
+		 * for this prepared transaction
+		 */
+		if (TwoPhaseExists(wait_xid))
+		{
+			List *foreign_trans = NIL;
+
+			foreign_trans = get_fdwxacts(MyDatabaseId, wait_xid, InvalidOid, InvalidOid,
+					 false, false, true);
+
+			if (foreign_trans == NIL)
+				return;
+			list_free(foreign_trans);
+		}
+	}
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction resolution.
+	 */
+	if (update_process_title)
+	{
+		int			len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status);
+		new_status[len] = '\0'; /* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The
+		 * latter would lead the client to believe that the distributed
+		 * transaction aborted, which is not true: it's already committed
+		 * locally. The former is no good either: the client has requested
+		 * committing a distributed transaction, and is entitled to assume
+		 * that a acknowledged commit is also commit on all foreign servers,
+		 * which might not be true. So in this case we issue a WARNING (which
+		 * some clients may be able to interpret) and shut off further output.
+		 * We do NOT reset PorcDiePending, so that the process will die after
+		 * the commit is cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve
+		 * them later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status);
+		pfree(new_status);
+	}
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC	   *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC	   *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC	   *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * AtEOXact_FdwXacts
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lc;
+
+	if (!is_commit)
+	{
+		foreach(lc, FdwXactParticipants)
+		{
+			FdwXactParticipant *fdw_part = lfirst(lc);
+
+			/*
+			 * If the foreign transaction has FdwXact entry we might have
+			 * prepared it. Skip already-prepared foreign transaction because
+			 * it has closed its transaction. But we are not sure that foreign
+			 * transaction with status == FDWXACT_STATUS_PREPARING has been
+			 * prepared or not. So we call the rollback API to close its
+			 * transaction for safety. The prepared foreign transaction that
+			 * we might have will be resolved by the foreign transaction
+			 * resolver.
+			 */
+			if (fdw_part->fdwxact)
+			{
+				bool		is_prepared;
+
+				LWLockAcquire(FdwXactLock, LW_SHARED);
+				is_prepared = fdw_part->fdwxact &&
+					fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED;
+				LWLockRelease(FdwXactLock);
+
+				if (is_prepared)
+					continue;
+			}
+
+			/* One-phase rollback foreign transaction */
+			FdwXactOnePhaseEndForeignTransaction(fdw_part, false);
+		}
+	}
+
+	/*
+	 * In commit cases, we have already prepared foreign transactions during
+	 * pre-commit phase. And these prepared transactions will be resolved by
+	 * the resolver process.
+	 */
+
+	ForgetAllFdwXactParticipants();
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Prepare foreign transactions.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	/*
+	 * We cannot prepare if any foreign server of participants isn't capable
+	 * of two-phase commit.
+	 */
+	if (is_foreign_twophase_commit_required(NULL) &&
+		(MyXactFlags & XACT_FLAGS_FDWNOPREPARE) != 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot prepare the transaction because some foreign servers involved in transaction can not prepare the transaction")));
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p)
+{
+	PGPROC	   *proc;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+			break;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+	{
+		*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+		*waitXid_p = proc->fdwXactWaitXid;
+	}
+	else
+	{
+		*nextResolutionTs_p = -1;
+		*waitXid_p = InvalidTransactionId;
+	}
+
+	LWLockRelease(FdwXactResolutionLock);
+
+	return proc;
+}
+
+/*
+ * Get one FdwXact entry to resolve. This function intended to be used when
+ * a resolver process get FdwXact entries to resolve. So we search entries
+ * while not including in-doubt transactions and in-progress transactions.
+ */
+static FdwXact
+get_fdwxact_to_resolve(Oid dbid, TransactionId xid)
+{
+	List	   *fdwxacts = NIL;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Don't include both in-doubt transactions and in-progress transactions */
+	fdwxacts = get_fdwxacts(dbid, xid, InvalidOid, InvalidOid,
+							false, false, false);
+
+	return fdwxacts == NIL ? NULL : (FdwXact) linitial(fdwxacts);
+}
+
+/*
+ * Resolve one distributed transaction on the given database . The target
+ * distributed transaction is fetched from the waiting queue and its transaction
+ * participants are fetched from the global array.
+ *
+ * Release the waiter and return true after we resolved the all of the foreign
+ * transaction participants. On failure, we re-enqueue the waiting backend after
+ * incremented the next resolution time.
+ */
+void
+FdwXactResolveTransaction(Oid dbid, TransactionId xid, PGPROC *waiter)
+{
+	FdwXact		fdwxact;
+
+	Assert(TransactionIdIsValid(xid));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	while ((fdwxact = get_fdwxact_to_resolve(MyDatabaseId, xid)) != NULL)
+	{
+		FdwXactRslvState *state;
+		ForeignServer *server;
+		UserMapping *usermapping;
+
+		CHECK_FOR_INTERRUPTS();
+
+		server = GetForeignServer(fdwxact->serverid);
+		usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+
+		state = create_fdwxact_state();
+		SpinLockAcquire(&fdwxact->mutex);
+		state->server = server;
+		state->usermapping = usermapping;
+		state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+		SpinLockRelease(&fdwxact->mutex);
+
+		FdwXactDetermineTransactionFate(fdwxact, false);
+
+		/* Do not hold during foreign transaction resolution */
+		LWLockRelease(FdwXactLock);
+
+		PG_TRY();
+		{
+			/*
+			 * Resolve the foreign transaction. When committing or aborting
+			 * prepared foreign transactions the previous status is always
+			 * FDWXACT_STATUS_PREPARED.
+			 */
+			FdwXactResolveForeignTransaction(fdwxact, state,
+											 FDWXACT_STATUS_PREPARED);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+			if (waiter->fdwXactState == FDWXACT_WAITING)
+			{
+				SHMQueueDelete(&(waiter->fdwXactLinks));
+				pg_write_barrier();
+				waiter->fdwXactNextResolutionTs =
+					TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+												foreign_xact_resolution_retry_interval);
+				FdwXactQueueInsert(waiter);
+			}
+			LWLockRelease(FdwXactResolutionLock);
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		elog(DEBUG2, "resolved one foreign transaction xid %u, serverid %d, userid %d",
+			 fdwxact->local_xid, fdwxact->serverid, fdwxact->userid);
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter could
+	 * already be detached if user cancelled to wait before resolution.
+	 */
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/*
+		 * Wake up the waiter only when we have set state and removed from
+		 * queue
+		 */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Determine whether the given foreign transaction should be committed or
+ * rolled back according to the result of the local transaction. This function
+ * changes fdwxact->status so the caller must hold FdwXactLock in exclusive
+ * mode or passing need_lock with true.
+ */
+static void
+FdwXactDetermineTransactionFate(FdwXact fdwxact, bool need_lock)
+{
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/*
+	 * The being resolved transaction must be either that has been cancelled
+	 * and marked as in-doubt or that has been prepared.
+	 */
+	Assert(fdwxact->indoubt ||
+		   fdwxact->status == FDWXACT_STATUS_PREPARED);
+
+	/*
+	 * If the local transaction is already committed, commit prepared foreign
+	 * transaction.
+	 */
+	if (TransactionIdDidCommit(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_COMMITTING;
+
+	/*
+	 * If the local transaction is already aborted, abort prepared foreign
+	 * transactions.
+	 */
+	else if (TransactionIdDidAbort(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+
+
+	/*
+	 * The local transaction is not in progress but the foreign transaction is
+	 * not prepared on the foreign server. This can happen when transaction
+	 * failed after registered this entry but before actual preparing on the
+	 * foreign server. So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(fdwxact->local_xid))
+		fdwxact->status = FDWXACT_STATUS_ABORTING;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is about
+	 * to be committed or aborted. This should not happen except for one case
+	 * where the local transaction is prepared and this foreign transaction is
+	 * being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot resolve the foreign transaction associated with in-progress transaction %u on server %u",
+						fdwxact->local_xid, fdwxact->serverid),
+				 errhint("The local transaction with xid %u might be prepared",
+						 fdwxact->local_xid)));
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Resolve the foreign transaction using the foreign data wrapper's transaction
+ * callback function. The 'state' is passed to the callback function. The fate of
+ * foreign transaction must be determined. If foreign transaction is resolved
+ * successfully, remove the FdwXact entry from the shared memory and also
+ * remove the corresponding on-disk file. If failed, the status of FdwXact
+ * entry changes to 'fallback_status' before erroring out.
+ */
+static void
+FdwXactResolveForeignTransaction(FdwXact fdwxact, FdwXactRslvState *state,
+								 FdwXactStatus fallback_status)
+{
+	ForeignServer *server;
+	ForeignDataWrapper *fdw;
+	FdwRoutine *fdw_routine;
+	bool		is_commit;
+
+	Assert(state != NULL);
+	Assert(state->server && state->usermapping && state->fdwxact_id);
+	Assert(fdwxact != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+		elog(ERROR, "cannot resolve foreign transaction whose fate is not determined");
+
+	is_commit = fdwxact->status == FDWXACT_STATUS_COMMITTING;
+	LWLockRelease(FdwXactLock);
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	fdw_routine = GetFdwRoutine(fdw->fdwhandler);
+
+	PG_TRY();
+	{
+		if (is_commit)
+			fdw_routine->CommitForeignTransaction(state);
+		else
+			fdw_routine->RollbackForeignTransaction(state);
+	}
+	PG_CATCH();
+	{
+		/* Back to the fallback status */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->status = fallback_status;
+		LWLockRelease(FdwXactLock);
+
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	/* Resolution was a success, remove the entry */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	elog(DEBUG1, "successfully %s the foreign transaction with xid %u db %u server %u user %u",
+		 is_commit ? "committed" : "rolled back",
+		 fdwxact->local_xid, fdwxact->dbid, fdwxact->serverid,
+		 fdwxact->userid);
+
+	fdwxact->status = FDWXACT_STATUS_RESOLVED;
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Return palloc'd and initialized FdwXactRslvState.
+ */
+static FdwXactRslvState *
+create_fdwxact_state(void)
+{
+	FdwXactRslvState *state;
+
+	state = palloc(sizeof(FdwXactRslvState));
+	state->server = NULL;
+	state->usermapping = NULL;
+	state->fdwxact_id = NULL;
+	state->flags = 0;
+
+	return state;
+}
+
+/*
+ * Return at least one FdwXact entry that matches to given argument,
+ * otherwise return NULL. All arguments must be valid values so that it can
+ * search exactly one (or none) entry. Note that this function intended to be
+ * used for modifying the returned FdwXact entry, so the caller must hold
+ * FdwXactLock in exclusive mode and it doesn't include the in-progress
+ * FdwXact entries.
+ */
+static FdwXact
+get_one_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	List	   *fdwxact_list;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* All search conditions must be valid values */
+	Assert(TransactionIdIsValid(xid));
+	Assert(OidIsValid(serverid));
+	Assert(OidIsValid(userid));
+	Assert(OidIsValid(dbid));
+
+	/* Include in-doubt transactions but don't include in-progress ones */
+	fdwxact_list = get_fdwxacts(dbid, xid, serverid, userid,
+								true, false, false);
+
+	/* Must be one entry since we search it by the unique key */
+	Assert(list_length(fdwxact_list) <= 1);
+
+	/* Could not find entry */
+	if (fdwxact_list == NIL)
+		return NULL;
+
+	return (FdwXact) linitial(fdwxact_list);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+FdwXactExists(Oid dbid, Oid serverid, Oid userid)
+{
+	List	   *fdwxact_list;
+	bool		ret = false;
+
+	/* Find entries from all FdwXact entries */
+	fdwxact_list = get_fdwxacts(dbid, InvalidTransactionId, serverid,
+								userid, true, true, true);
+
+	if (fdwxact_list != NIL)
+		ret = true;
+
+	list_free(fdwxact_list);
+	return ret;
+}
+
+/*
+ * Returns an array of all foreign prepared transactions for the user-level
+ * function pg_foreign_xacts, and the number of entries to num_p.
+ *
+ * WARNING -- we return even those transactions whose information is not
+ * completely filled yet. The caller should filter them out if he doesn't
+ * want them.
+ *
+ * The returned array is palloc'd.
+ */
+static FdwXact
+get_all_fdwxacts(int *num_p)
+{
+	List	   *all_fdwxacts;
+	ListCell   *lc;
+	FdwXact		fdwxacts;
+	int			num_fdwxacts = 0;
+
+	Assert(num_p != NULL);
+
+	/* Get all entries */
+	all_fdwxacts = get_fdwxacts(InvalidOid, InvalidTransactionId,
+								InvalidOid, InvalidOid, true,
+								true, true);
+
+	if (all_fdwxacts == NIL)
+	{
+		*num_p = 0;
+		return NULL;
+	}
+
+	fdwxacts = (FdwXact)
+		palloc(sizeof(FdwXactData) * list_length(all_fdwxacts));
+	*num_p = list_length(all_fdwxacts);
+
+	/* Convert list to array of FdwXact */
+	foreach(lc, all_fdwxacts)
+	{
+		FdwXact		fx = (FdwXact) lfirst(lc);
+
+		memcpy(fdwxacts + num_fdwxacts, fx,
+			   sizeof(FdwXactData));
+		num_fdwxacts++;
+	}
+
+	list_free(all_fdwxacts);
+
+	return fdwxacts;
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respectively.
+ */
+static List *
+get_fdwxacts(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			 bool include_indoubt, bool include_in_progress, bool need_lock)
+{
+	int			i;
+	List	   *fdwxact_list = NIL;
+
+	if (need_lock)
+		LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* include in-doubt transaction? */
+		if (!include_indoubt && fdwxact->indoubt)
+			continue;
+
+		/* include in-progress transaction? */
+		if (!include_in_progress && FdwXactIsBeingResolved(fdwxact))
+			continue;
+
+		/* Append it if matched */
+		fdwxact_list = lappend(fdwxact_list, fdwxact);
+	}
+
+	if (need_lock)
+		LWLockRelease(FdwXactLock);
+
+	return fdwxact_list;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record in
+		 * FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transaction log record", info);
+
+	return;
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	   *id;
+	int			id_len = 0;
+
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char		buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		/*
+		 * FDW doesn't provide the callback function, generate an unique
+		 * identifier.
+		 */
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+				 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifier \"%s\" is too long",
+						id),
+				 errdetail("Foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint's redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;					/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence of long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked invalid,
+	 * because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+				(errmsg_plural("%u foreign transaction state file was written "
+							   "for long-running prepared transactions",
+							   "%u foreign transaction state files were written "
+							   "for long-running prepared transactions",
+							   serialized_fdwxacts,
+							   serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
+									&read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+				 errdetail("Failed while allocating an XLog reading processor.")));
+	XLogBeginRead(xlogreader, lsn);
+	record = XLogReadRecord(xlogreader, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not read foreign transaction state from xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not recreate foreign transaction state file \"%s\": %m",
+						path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId origNextXid =
+	XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	   *buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open FDW transaction state file \"%s\": %m",
+						path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+	int			i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char	   *buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The status of
+	 * the transaction is set as preparing, since we do not know the exact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction which prepared this foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							 fdwxact_data->serverid, fdwxact_data->userid,
+							 fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know the xact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;		/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	fdwxact = get_one_fdwxact(dbid, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		return;
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	int			i;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign prepared transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+check_foreign_twophase_commit(int *newval, void **extra, GucSource source)
+{
+	ForeignTwophaseCommitLevel newForeignTwophaseCommitLevel = *newval;
+
+	/* Parameter check */
+	if (newForeignTwophaseCommitLevel > FOREIGN_TWOPHASE_COMMIT_DISABLED &&
+		(max_prepared_foreign_xacts == 0 || max_foreign_xact_resolvers == 0))
+	{
+		GUC_check_errdetail("Cannot enable \"foreign_twophase_commit\" when "
+							"\"max_prepared_foreign_transactions\" or \"max_foreign_transaction_resolvers\""
+							"is zero value");
+		return false;
+	}
+
+	return true;
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+} WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	7
+	FuncCallContext *funcctx;
+	WorkingStatus *status;
+	char	   *xact_status;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		TupleDesc	tupdesc;
+		MemoryContext oldcontext;
+		int			num_fdwxacts = 0;
+
+		/* create a function context for cross-call persistence */
+		funcctx = SRF_FIRSTCALL_INIT();
+
+		/*
+		 * Switch to memory context appropriate for multiple function calls
+		 */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		/* build tupdesc for result tuples */
+		/* this had better match pg_fdwxacts view in system_views.sql */
+		tupdesc = CreateTemplateTupleDesc(PG_PREPARED_FDWXACTS_COLS);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction",
+						   XIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 6, "indoubt",
+						   BOOLOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 7, "identifier",
+						   TEXTOID, -1, 0);
+
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/*
+		 * Collect status information that we will format and send out as a
+		 * result set.
+		 */
+		status = (WorkingStatus *) palloc(sizeof(WorkingStatus));
+		funcctx->user_fctx = (void *) status;
+
+		status->fdwxacts = get_all_fdwxacts(&num_fdwxacts);
+		status->num_xacts = num_fdwxacts;
+		status->cur_xact = 0;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	status = funcctx->user_fctx;
+
+	while (status->cur_xact < status->num_xacts)
+	{
+		FdwXact		fdwxact = &status->fdwxacts[status->cur_xact++];
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+		HeapTuple	tuple;
+		Datum		result;
+
+		if (!fdwxact->valid)
+			continue;
+
+		/*
+		 * Form tuple with appropriate data.
+		 */
+		MemSet(values, 0, sizeof(values));
+		MemSet(nulls, 0, sizeof(nulls));
+
+		values[0] = ObjectIdGetDatum(fdwxact->dbid);
+		values[1] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[2] = ObjectIdGetDatum(fdwxact->serverid);
+		values[3] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (fdwxact->status)
+		{
+			case FDWXACT_STATUS_INITIAL:
+				xact_status = "initial";
+				break;
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			case FDWXACT_STATUS_RESOLVED:
+				xact_status = "resolved";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[4] = CStringGetTextDatum(xact_status);
+		values[5] = BoolGetDatum(fdwxact->indoubt);
+		values[6] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		result = HeapTupleGetDatum(tuple);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Built-in function to resolve a prepared foreign transaction manually.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			serverid = PG_GETARG_OID(1);
+	Oid			userid = PG_GETARG_OID(2);
+	ForeignServer *server;
+	UserMapping *usermapping;
+	FdwXact		fdwxact;
+	FdwXactRslvState *state;
+	FdwXactStatus prev_status;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	server = GetForeignServer(serverid);
+	usermapping = GetUserMapping(userid, serverid);
+	state = create_fdwxact_state();
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+	{
+		LWLockRelease(FdwXactLock);
+		PG_RETURN_BOOL(false);
+	}
+
+	state->server = server;
+	state->usermapping = usermapping;
+	state->fdwxact_id = pstrdup(fdwxact->fdwxact_id);
+
+	SpinLockAcquire(&fdwxact->mutex);
+	prev_status = fdwxact->status;
+	SpinLockRelease(&fdwxact->mutex);
+
+	FdwXactDetermineTransactionFate(fdwxact, false);
+
+	ereport(LOG,
+			(errmsg("trying to %s the foreign transaction associated with transaction %u on server %u",
+					fdwxact->status == FDWXACT_STATUS_COMMITTING?"COMMIT":"ABORT",
+					fdwxact->local_xid, fdwxact->serverid)));
+
+	LWLockRelease(FdwXactLock);
+
+	FdwXactResolveForeignTransaction(fdwxact, state, prev_status);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			serverid = PG_GETARG_OID(1);
+	Oid			userid = PG_GETARG_OID(2);
+	FdwXact		fdwxact;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_one_fdwxact(MyDatabaseId, xid, serverid, userid);
+
+	if (fdwxact == NULL)
+		PG_RETURN_BOOL(false);
+
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000000..e293d13562
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,641 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to retry resolution.
+ */
+void
+FdwXactLauncherRequestToLaunchForRetry(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		SetLatch(FdwXactRslvCtl->launcher_latch);
+}
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool		found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int			slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			resolver->pid = InvalidPid;
+			resolver->dbid = InvalidOid;
+			resolver->in_use = false;
+			resolver->last_resolved_time = 0;
+			resolver->latch = NULL;
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == 0);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz now;
+		long		wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int			rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a
+		 * foreign_xact_resolution_retry_interval but always starts when the
+		 * backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool		launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested but not
+			 * running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we should
+			 * retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool		found = false;
+	int			i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the same
+	 * database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find
+		 * FdwXact entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int			unused_slot = -1;
+	int			i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign transaction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to
+	 * wait until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	   *resolver_dbs;	/* DBs resolver's running on */
+	HTAB	   *fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL		ctl;
+	HASH_SEQ_STATUS status;
+	Oid		   *entry;
+	bool		launched;
+	int			i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->indoubt)
+			continue;
+
+		hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+		return false;
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool		found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	return launched;
+}
+
+/* Register a background worker running the foreign transaction launcher */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid			dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int			i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign transaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int			rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		/* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+					   10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Returns activity of all foreign transaction resolvers.
+ */
+Datum
+pg_stat_get_foreign_xact(PG_FUNCTION_ARGS)
+{
+#define PG_STAT_GET_FDWXACT_RESOLVERS_COLS 3
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+	int			i;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not " \
+						"allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+		pid_t		pid;
+		Oid			dbid;
+		TimestampTz last_resolved_time;
+		Datum		values[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+		bool		nulls[PG_STAT_GET_FDWXACT_RESOLVERS_COLS];
+
+
+		SpinLockAcquire(&(resolver->mutex));
+		if (resolver->pid == InvalidPid)
+		{
+			SpinLockRelease(&(resolver->mutex));
+			continue;
+		}
+
+		pid = resolver->pid;
+		dbid = resolver->dbid;
+		last_resolved_time = resolver->last_resolved_time;
+		SpinLockRelease(&(resolver->mutex));
+
+		memset(nulls, 0, sizeof(nulls));
+		/* pid */
+		values[0] = Int32GetDatum(pid);
+
+		/* dbid */
+		values[1] = ObjectIdGetDatum(dbid);
+
+		/* last_resolved_time */
+		if (last_resolved_time == 0)
+			nulls[2] = true;
+		else
+			values[2] = TimestampTzGetDatum(last_resolved_time);
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000000..4843aeacc9
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,343 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int			foreign_xact_resolution_retry_interval;
+int			foreign_xact_resolver_timeout = 60 * 1000;
+bool		foreign_xact_resolve_indoubt_xacts;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+	MyFdwXactResolver->last_resolved_time = 0;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int			slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		PGPROC	   *waiter = NULL;
+		TransactionId waitXid = InvalidTransactionId;
+		TimestampTz resolutionTs = -1;
+		int			rc;
+		TimestampTz now;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or got the waiter
+		 * that has future resolution time.
+		 */
+		while ((waiter = FdwXactGetWaiter(&resolutionTs, &waitXid)) != NULL)
+		{
+			CHECK_FOR_INTERRUPTS();
+			Assert(TransactionIdIsValid(waitXid));
+
+			if (resolutionTs > now)
+				break;
+
+			elog(DEBUG2, "resolver got one waiter with xid %u", waitXid);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveTransaction(MyDatabaseId, waitXid, waiter);
+			CommitTransactionCommand();
+
+			/* Update my stats */
+			SpinLockAcquire(&(MyFdwXactResolver->mutex));
+			MyFdwXactResolver->last_resolved_time = GetCurrentTimestamp();
+			SpinLockRelease(&(MyFdwXactResolver->mutex));
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz last_resolved_time;
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	last_resolved_time = MyFdwXactResolver->last_resolved_time;
+	timeout = TimestampTzPlusMilliseconds(last_resolved_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * checked FdwXactWaiterExists.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long		sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(MyFdwXactResolver->last_resolved_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index f88d72fd86..982c1a36cc 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -13,6 +13,7 @@ OBJS = \
 	clogdesc.o \
 	committsdesc.o \
 	dbasedesc.o \
+	fdwxactdesc.o \
 	genericdesc.o \
 	gindesc.o \
 	gistdesc.o \
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000000..ca761763e5
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxactdesc.c
+ *		PostgreSQL global transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdwxactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdwxact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		FdwXactOnDiskData *fdwxact_insert = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_insert->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_insert->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_insert->dbid);
+		appendStringInfo(buf, " local xid: %u,", fdwxact_insert->local_xid);
+		appendStringInfo(buf, " id: %s", fdwxact_insert->fdwxact_id);
+	}
+	else
+	{
+		xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_remove->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_remove->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_remove->dbid);
+		appendStringInfo(buf, " local xid: %u", fdwxact_remove->xid);
+	}
+
+}
+
+const char *
+fdwxact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDWXACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDWXACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 1cd97852e8..ea045174e0 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 58091f6b52..200cf9d067 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -10,6 +10,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 5adf956f41..e8e6a5e2b5 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -850,6 +851,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 	return result;
 }
 
+/*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
 /*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
@@ -2263,6 +2293,12 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, true);
 }
 
 /*
@@ -2322,6 +2358,12 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	FdwXactWaitToBeResolved(xid, false);
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index e3c60f23cd..405271387d 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1218,6 +1219,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1226,6 +1228,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1264,12 +1267,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1427,6 +1431,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitToBeResolved(xid, true);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2086,6 +2098,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2246,6 +2261,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2333,6 +2349,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2527,6 +2545,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2732,6 +2751,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7621fc05e2..db116ff7ca 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
@@ -4545,6 +4546,7 @@ InitControlFile(uint64 sysidentifier)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6226,6 +6228,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_wal_senders",
 									 max_wal_senders,
 									 ControlFile->max_wal_senders);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
@@ -6768,14 +6773,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -6967,7 +6973,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7480,6 +7489,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7810,6 +7820,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9086,6 +9099,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9519,8 +9533,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9538,6 +9554,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9554,6 +9571,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9759,6 +9777,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -9958,6 +9977,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 5a6dc61630..246c3df966 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -333,6 +333,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
@@ -826,6 +829,14 @@ CREATE VIEW pg_stat_subscription AS
             LEFT JOIN pg_stat_get_subscription(NULL) st
                       ON (st.subid = su.oid);
 
+CREATE VIEW pg_stat_foreign_xact AS
+    SELECT
+            r.pid,
+            r.dbid,
+            r.last_resolved_time
+    FROM pg_stat_get_foreign_xact() r
+    WHERE r.pid IS NOT NULL;
+
 CREATE VIEW pg_stat_ssl AS
     SELECT
             S.pid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index fbde9f88e7..acc9a86642 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2859,8 +2859,14 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
+	}
 
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index f197869752..6206265424 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1101,6 +1103,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (FdwXactExists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1419,6 +1433,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
+	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (FdwXactExists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
 	/*
 	 * Do the deletion
 	 */
@@ -1572,6 +1595,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index fb6ce49056..3fa8bfe09f 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/table.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
@@ -939,7 +940,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 513471ab9b..29f376e48c 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,9 +226,31 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
+	}
 
 	return scanstate;
 }
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 870a7428f1..d82d32ecb4 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -2411,6 +2413,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index 61e48ca3f8..8f411c0559 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMappingOid - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,18 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction && !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction && routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper must support both commit and rollback routines or neither");
+
+	if (routine->PrepareForeignTransaction &&
+		!routine->CommitForeignTransaction &&
+		!routine->RollbackForeignTransaction)
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 6c684b5e12..39c1b08699 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -12,6 +12,8 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -128,6 +130,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 4763c24be9..6ad744db80 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3593,6 +3593,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3799,6 +3805,11 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDWXACT:
+			event_name = "FdwXact";
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -4023,6 +4034,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 73d278f3b2..0c3f999e0e 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -909,6 +911,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -973,12 +979,13 @@ PostmasterMain(int argc, char *argv[])
 	LocalProcessControlFile(false);
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index c2e5e3abf8..9d34817f39 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -151,6 +151,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 427b0d59cd..55609eed81 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(void)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(void)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index f45a619deb..6ff1a6758c 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -94,6 +94,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -249,6 +251,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1313,6 +1316,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1378,6 +1382,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1427,6 +1432,15 @@ GetOldestXmin(Relation rel, int flags)
 		NormalTransactionIdPrecedes(replication_slot_xmin, result))
 		result = replication_slot_xmin;
 
+	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
 	/*
 	 * After locks have been released and vacuum_defer_cleanup_age has been
 	 * applied, check whether we need to back up further to make logical
@@ -3129,6 +3143,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations of the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index db47843229..adb276370c 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -49,3 +49,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 CLogTruncationLock					44
+FdwXactLock							45
+FdwXactResolverLock					46
+FdwXactResolutionLock				47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 9938cddb57..71e74e7448 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -421,6 +422,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* Initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -822,6 +827,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index cb8c23e4b7..a4472fcb60 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3034,6 +3036,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index af876d1f01..3e1c505e28 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -425,6 +426,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 	{NULL, 0, false}
 };
 
+/*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
 /*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
@@ -761,6 +781,12 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
+	/* FDWXACT_RESOLVER */
+	gettext_noop("Foreign Transaction Management / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2469,6 +2495,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FDWXACT_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FDWXACT_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4531,6 +4603,16 @@ static struct config_enum ConfigureNamesEnum[] =
 		NULL, assign_synchronous_commit, NULL
 	},
 
+	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		check_foreign_twophase_commit, NULL, NULL
+	},
+
 	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index aa44f0c9bf..c6a302e49a 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -342,6 +344,20 @@
 #max_sync_workers_per_subscription = 2	# taken from max_logical_replication_workers
 
 
+#------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+#foreign_twophase_commit = disabled	# use two-phase commit for distributed transactions
+					# disabled prefer or required
+
 #------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index a0b0458108..8701c5f005 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index a6577486ce..f520a7a235 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -208,6 +208,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index e73639df74..3041c39bc0 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -302,6 +302,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 233441837f..b040202043 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -710,6 +710,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -914,6 +915,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/fdwxactdesc.c b/src/bin/pg_waldump/fdwxactdesc.c
new file mode 100644
index 0000000000..ce8c21880c
--- /dev/null
+++ b/src/bin/pg_waldump/fdwxactdesc.c
@@ -0,0 +1 @@
+../../../src/backend/access/rmgrdesc/fdwxactdesc.c
\ No newline at end of file
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca4b1..b616cea347 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000000..dd8433f42c
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,167 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL global transaction manager
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDWXACT_H
+#define FDWXACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* fdwXactState */
+#define	FDWXACT_NOT_WAITING		0
+#define	FDWXACT_WAITING			1
+#define	FDWXACT_WAIT_COMPLETE	2
+
+/* Flag passed to FDW transaction management APIs */
+#define FDWXACT_FLAG_ONEPHASE		0x01	/* transaction can commit/rollback
+											 * without preparation */
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER, /* use twophase commit where available */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support
+										 * twophase commit */
+} ForeignTwophaseCommitLevel;
+
+/* Enum to track the status of foreign transaction */
+typedef enum
+{
+	FDWXACT_STATUS_INVALID = 0,
+	FDWXACT_STATUS_INITIAL,
+	FDWXACT_STATUS_PREPARING,	/* foreign transaction is being prepared */
+	FDWXACT_STATUS_PREPARED,	/* foreign transaction is prepared */
+	FDWXACT_STATUS_COMMITTING,	/* foreign prepared transaction is being
+								 * committed */
+	FDWXACT_STATUS_ABORTING,	/* foreign prepared transaction is being
+								 * aborted */
+	FDWXACT_STATUS_RESOLVED
+} FdwXactStatus;
+
+typedef struct FdwXactData *FdwXact;
+
+/*
+ * Shared memory state of a single foreign transaction.
+ */
+typedef struct FdwXactData
+{
+	FdwXact		fdwxact_free_next;	/* Next free FdwXact entry */
+
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	TransactionId local_xid;	/* XID of local transaction */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	bool		indoubt;		/* Is an in-doubt transaction? */
+	slock_t		mutex;			/* Protect the above fields */
+
+	/* The status of the foreign transaction, protected by FdwXactLock */
+	FdwXactStatus status;
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;	/* XLOG offset of inserting this entry
+									 * start */
+	XLogRecPtr	insert_end_lsn; /* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to
+								 * file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* prepared transaction
+												 * identifier */
+}			FdwXactData;
+
+/*
+ * Shared memory layout for maintaining foreign prepared transaction entries.
+ * Adding or removing FdwXact entry needs to hold FdwXactLock in exclusive mode,
+ * and iterating fdwXacts needs that in shared mode.
+ */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		free_fdwxacts;
+
+	/* Number of valid foreign transaction entries */
+	int			num_fdwxacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdwxacts[FLEXIBLE_ARRAY_MEMBER];	/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* State data for foreign transaction resolution, passed to FDW callbacks */
+typedef struct FdwXactRslvState
+{
+	/* Foreign transaction information */
+	char	   *fdwxact_id;
+
+	ForeignServer *server;
+	UserMapping *usermapping;
+
+	int			flags;			/* OR of FDWXACT_FLAG_xx flags */
+} FdwXactRslvState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+/* Function declarations */
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void RegisterFdwXactByRelId(Oid relid, bool modified);
+extern void RegisterFdwXactByServerId(Oid serverid, bool modified);
+extern void PreCommit_FdwXacts(void);
+extern void FdwXactResolveTransaction(Oid dbid, TransactionId xid, PGPROC *waiter);
+extern bool FdwXactResolveInDoubtTransactions(Oid dbid);
+extern void FdwXactWaitToBeResolved(TransactionId wait_xid, bool commit);
+extern PGPROC *FdwXactGetWaiter(TimestampTz *nextResolutionTs_p, TransactionId *waitXid_p);
+extern bool FdwXactWaiterExists(Oid dbid);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void KnownFdwXactRecreateFiles(XLogRecPtr redo_horizon);
+extern void RecoverFdwXacts(void);
+extern bool FdwXactExists(Oid dboid, Oid serverid, Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwTwoPhaseNeeded(void);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+extern void FdwXactMarkForeignServerAccessed(Oid relid, bool modified);
+extern bool check_foreign_twophase_commit(int *newval, void **extra,
+										  GucSource source);
+
+#endif							/* FDWXACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000000..c3ed1ecfaf
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,29 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherRequestToLaunch(void);
+extern void FdwXactLauncherRequestToLaunchForRetry(void);
+extern void FdwXactLaunchOrWakeupResolver(void);
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+
+
+#endif							/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000000..779848113c
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int	foreign_xact_resolver_timeout;
+
+#endif							/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000000..b4cec76eae
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,54 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDWXACT_INSERT	0x00
+#define XLOG_FDWXACT_REMOVE	0x10
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDWXACT_ID_MAX_LEN 200
+
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdwxact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+	bool		force;
+} xl_fdwxact_remove;
+
+extern void fdwxact_redo(XLogReaderState *record);
+extern void fdwxact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdwxact_identify(uint8 info);
+
+#endif							/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000000..80691b5c07
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,66 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/resolver_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t		pid;			/* this resolver's PID, or 0 if not active */
+	Oid			dbid;			/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool		in_use;
+
+	/* Stats */
+	TimestampTz last_resolved_time;
+
+	/* Protect shared variables shown above */
+	slock_t		mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	   *latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/* Foreign transaction resolution queue. Protected by FdwXactLock */
+	SHM_QUEUE	fdwxact_queue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch	   *launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+#define SizeOfFdwXactRslvCtlData \
+	(offsetof(FdwXactRslvCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif							/* RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 6c15df7e70..177b236f70 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Foreign Transactions", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 2ca71c3445..bd027a2861 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index a04fc70326..6f1f336e31 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -108,6 +108,13 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
+/*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 27ded593ab..f15a802e5c 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -232,6 +232,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index de5670e538..9884f5f8e7 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 87d25d4a4b..33c6df1375 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5214,6 +5214,13 @@
   proargmodes => '{i,o,o,o,o,o,o,o,o}',
   proargnames => '{subid,subid,relid,pid,received_lsn,last_msg_send_time,last_msg_receipt_time,latest_end_lsn,latest_end_time}',
   prosrc => 'pg_stat_get_subscription' },
+{ oid => '9705', descr => 'statistics: information about foreign transaction resolver',
+  proname => 'pg_stat_get_foreign_xact', proisstrict => 'f', provolatile => 's',
+  proparallel => 'r', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,oid,timestamptz}',
+  proargmodes => '{o,o,o}',
+  proargnames => '{pid,dbid,last_resolved_time}',
+  prosrc => 'pg_stat_get_foreign_xact' },
 { oid => '2026', descr => 'statistics: current backend PID',
   proname => 'pg_backend_pid', provolatile => 's', proparallel => 'r',
   prorettype => 'int4', proargtypes => '', prosrc => 'pg_backend_pid' },
@@ -5927,6 +5934,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '9706', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{oid,xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o,o}',
+  proargnames => '{dbid,xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '9707', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '9708', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -6045,6 +6070,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '9709',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 95556dfb15..8d046cc4e4 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 5e0cf533fb..5596ee591c 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -69,6 +69,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 											   bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 														 bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index a07012bf4b..48602acd7b 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -758,6 +758,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -836,7 +838,9 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDWXACT,
+	WAIT_EVENT_FDWXACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -919,6 +923,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index d21780108b..35ffbbca93 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -16,6 +16,7 @@
 
 #include "access/clog.h"
 #include "access/xlogdefs.h"
+#include "datatype/timestamp.h"
 #include "lib/ilist.h"
 #include "storage/latch.h"
 #include "storage/lock.h"
@@ -152,6 +153,16 @@ struct PGPROC
 	int			syncRepState;	/* wait state for sync rep */
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
+	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
 	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index a5c7d0c064..0f73b64937 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -125,4 +127,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 454c2df487..6010dbcdee 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,9 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
+	FDWXACT_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index a2077bbad4..3cc765a496 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1342,6 +1342,14 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.dbid,
+    f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(dbid, xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
@@ -1848,6 +1856,11 @@ pg_stat_database_conflicts| SELECT d.oid AS datid,
     pg_stat_get_db_conflict_bufferpin(d.oid) AS confl_bufferpin,
     pg_stat_get_db_conflict_startup_deadlock(d.oid) AS confl_deadlock
    FROM pg_database d;
+pg_stat_foreign_xact| SELECT r.pid,
+    r.dbid,
+    r.last_resolved_time
+   FROM pg_stat_get_foreign_xact() r(pid, dbid, last_resolved_time)
+  WHERE (r.pid IS NOT NULL);
 pg_stat_gssapi| SELECT s.pid,
     s.gss_auth AS gss_authenticated,
     s.gss_princ AS principal,
-- 
2.21.1 (Apple Git-122.3)

From 06c686d4fec361666751fb0cf9f047c26c0ef1d8 Mon Sep 17 00:00:00 2001
From: Muhammad Usama <m.usama@highgo.ca>
Date: Thu, 26 Mar 2020 14:12:17 +0500
Subject: [PATCH v19 1/5] Keep track of writing on non-temporary relation

Authors: Muhammad Usama, Masahiko Sawada, Ahutosh Bapat
---
 src/backend/executor/nodeModifyTable.c | 16 ++++++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 22 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index d71c0a4322..870a7428f1 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -574,6 +574,10 @@ ExecInsert(ModifyTableState *mtstate,
 										   NULL,
 										   specToken);
 
+			/* Make note that we've wrote on non-temporary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -612,6 +616,10 @@ ExecInsert(ModifyTableState *mtstate,
 							   estate->es_output_cid,
 							   0, NULL);
 
+			/* Make note that we've wrote on non-temporary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -963,6 +971,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temporary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1475,6 +1487,10 @@ lreplace:;
 	if (canSetTag)
 		(estate->es_processed)++;
 
+	/* Make note that we've wrote on non-temporary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/* AFTER ROW UPDATE Triggers */
 	ExecARUpdateTriggers(estate, resultRelInfo, tupleid, oldtuple, slot,
 						 recheckIndexes,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7ee04babc2..a04fc70326 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -102,6 +102,12 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
-- 
2.21.1 (Apple Git-122.3)

From 49ece6bd9a68c49d7279f86493f6c3c2091fcfad Mon Sep 17 00:00:00 2001
From: Muhammad Usama <m.usama@highgo.ca>
Date: Thu, 26 Mar 2020 21:27:18 +0500
Subject: [PATCH v19 3/5] Documentation update.

Authors: Muhammad Usama, Masahiko Sawada, Ahutosh Bapat
---
 doc/src/sgml/catalogs.sgml                | 145 +++++++++++++
 doc/src/sgml/config.sgml                  | 146 ++++++++++++-
 doc/src/sgml/distributed-transaction.sgml | 158 +++++++++++++++
 doc/src/sgml/fdwhandler.sgml              | 236 ++++++++++++++++++++++
 doc/src/sgml/filelist.sgml                |   1 +
 doc/src/sgml/func.sgml                    |  89 ++++++++
 doc/src/sgml/monitoring.sgml              |  60 ++++++
 doc/src/sgml/postgres.sgml                |   1 +
 doc/src/sgml/storage.sgml                 |   6 +
 9 files changed, 841 insertions(+), 1 deletion(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 64614b569c..af1c2dcbcf 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8192,6 +8192,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
       <entry>open cursors</entry>
      </row>
 
+     <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
      <row>
       <entry><link linkend="view-pg-file-settings"><structname>pg_file_settings</structname></link></entry>
       <entry>summary of configuration file contents</entry>
@@ -9650,6 +9655,146 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with that this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on that the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>initial</literal> : Initial status.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>resolved</literal> : This foreign transaction has been resolved.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-dbout status and
+       needs to be resolved by calling <function>pg_resolve_fdwxact</function>
+       function.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 355b408b0a..02849939ef 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4426,7 +4426,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -8928,6 +8927,151 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether transaction commit will wait for all involving foreign
+         transaction to be resolved before the command returns a "success"
+         indication to the client. Valid values are <literal>required</literal>,
+         <literal>prefer</literal> and <literal>disabled</literal>. The default
+         setting is <literal>disabled</literal>. Setting to
+         <literal>disabled</literal> don't use two-phase commit protocol to
+         commit or rollback distributed transactions. When set to
+         <literal>required</literal> the distributed transaction strictly
+         requires that all written servers can use two-phase commit protocol.
+         That is, the distributed transaction cannot commit if even one server
+         does not support the transaction management callback routines
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. Note that when <literal>disabled</literal> or
+         <literal>prefer</literal> there can be risk of database consistency
+         among all servers that involved in the distributed transaction when some
+         foreign server crashes during committing the distributed transaction.
+        </para>
+
+        <para>
+         Both <varname>max_prepared_foreign_transactions</varname> and
+         <varname>max_foreign_transaction_resolvers</varname> must be non-zero
+         value to set this parameter either <literal>required</literal> or
+         <literal>prefer</literal>.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000000..350b1afe68
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,158 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatially,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).
+    A <productname>PostgreSQL</productname> server that received SQL is called
+    <firstterm>coordinator node</firstterm> who is responsible for coordinating
+    all the partipanting transactions. Using two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed
+    transaction manager prepares all transaction on the foreign servers if
+    two-phase commit is required. Two-phase commit is required when the
+    transaction modifies data on two or more servers including the local server
+    itself and <xref linkend="guc-foreign-twophase-commit"/>is
+    <literal>required</literal> or <literal>prefer</literal>. If all preparations
+    on foreign servers got successful go to the next step. Any failure happens
+    in this step <productname>PostgreSQL</productname> changes to rollback, then
+    rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the
+    transaction locally. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes rollback, then rollback all
+    transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolution">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that is
+    responsible for foreign transaction resolution. They commit or rollback all
+    prepared transaction on foreign servers if the coordinator received agreement
+    messages from all foreign servers during the first step.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on one database of the coordinator side. On failure during resolution, they
+    retries to resolve at an interval of
+    <varname>foreign_transaction_resolution_interval</varname> time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped. So to drop the database, you can call
+     <function>pg_stop_foreign_xact_resovler</function> function before dropping
+     the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>Manual Resolution of In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, distributed transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the coordinator node crashed during either preparing or
+    resolving distributed transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="pg-stat-foreign-xact-view"/>
+    view. These foreign transactions need to be resolved by using
+    <function>pg_resolve_foriegn_xact</function> function.
+    <productname>PostgreSQL</productname> doesn't have facilities to automatially
+    resolve in-doubt transactions. These behavior might change in a future release.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-monitoring">
+   <title>Monitoring</title>
+   <para>
+    The monitoring information about foreign transaction resolvers is visible in
+    <link linkend="pg-stat-foreign-xact-view"><literal>pg_stat_foreign_xact</literal></link>
+    view. This view contains one row for every foreign transaction resolver worker.
+   </para>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 6587678af2..dd0358ef22 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1415,6 +1415,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1894,4 +2015,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 3da2365ea9..80a87fa5d1 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 7a0bb0c70a..aed898248a 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -21849,6 +21849,95 @@ SELECT (pg_stat_file('filename')).modification;
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 270178d57e..cbc57d3c12 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -384,6 +384,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_foreign_xact</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1264,6 +1272,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry><literal>CheckpointerMain</literal></entry>
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
+        <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
         <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
@@ -1491,6 +1511,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry><literal>SafeSnapshot</literal></entry>
          <entry>Waiting for a snapshot for a <literal>READ ONLY DEFERRABLE</literal> transaction.</entry>
         </row>
+        <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
         <row>
          <entry><literal>SyncRep</literal></entry>
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
@@ -2415,6 +2439,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-view" xreflabel="pg_stat_foreign_xact">
+   <title><structname>pg_stat_foreign_xact</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index e59cba7997..dee3f72f7e 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -163,6 +163,7 @@
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 1c19e863d2..3f4c806ed1 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -83,6 +83,12 @@ Item
   subsystem</entry>
 </row>
 
+<row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
 <row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
-- 
2.21.1 (Apple Git-122.3)

From 49d46cd05af910b2613d85427e1a5b539034f7af Mon Sep 17 00:00:00 2001
From: Muhammad Usama <m.usama@highgo.ca>
Date: Thu, 26 Mar 2020 21:28:58 +0500
Subject: [PATCH v19 4/5] postgres_fdw supports atomic commit APIs.

Authors: Muhammad Usama, Masahiko Sawada, Ahutosh Bapat
---
 contrib/postgres_fdw/Makefile                 |   7 +-
 contrib/postgres_fdw/connection.c             | 603 +++++++++++-------
 .../postgres_fdw/expected/postgres_fdw.out    | 265 +++++++-
 contrib/postgres_fdw/fdwxact.conf             |   3 +
 contrib/postgres_fdw/postgres_fdw.c           |  21 +-
 contrib/postgres_fdw/postgres_fdw.h           |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql     | 120 +++-
 doc/src/sgml/postgres-fdw.sgml                |  45 ++
 8 files changed, 822 insertions(+), 249 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index ee8a80a392..91fa6e39fc 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -16,7 +16,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -29,3 +29,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index e45647f3ea..04410c27fd 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2020, PostgreSQL Global Development Group
  *
@@ -12,6 +12,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "catalog/pg_user_mapping.h"
@@ -56,6 +57,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -69,17 +71,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 								   SubTransactionId mySubid,
 								   SubTransactionId parentSubid,
@@ -92,23 +90,26 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 									 PGresult **result);
 static bool UserMappingPasswordRequired(UserMapping *user);
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +129,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,12 +136,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -155,6 +149,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -182,6 +191,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +200,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +211,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,11 +227,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -472,7 +520,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -699,193 +747,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 	PG_END_TRY();
 }
 
-/*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow any remote transactions, since it's not
-					 * very reasonable to hold them open until the prepared
-					 * transaction is committed.  For the moment, throw error
-					 * unconditionally; later we might allow read-only cases.
-					 * Note that the error will cause us to come right back
-					 * here with event == XACT_EVENT_ABORT, so we'll clean up
-					 * the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot PREPARE a transaction that has operated on postgres_fdw foreign tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
 /*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
@@ -902,10 +763,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -916,6 +773,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1250,3 +1111,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recusively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 62c2697920..cd871fe314 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -191,15 +210,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8961,16 +8982,226 @@ DROP OWNED BY regress_nosuper;
 DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
--- Two-phase transactions are not supported.
+
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
 BEGIN;
-SELECT count(*) FROM ft1;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
  count 
 -------
-   822
+     0
 (1 row)
 
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
-ERROR:  cannot PREPARE a transaction that has operated on postgres_fdw foreign tables
-ROLLBACK;
-WARNING:  there is no transaction in progress
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000000..3fdbf93cdb
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 2175dff824..0873d1d4b7 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include <limits.h>
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -504,7 +505,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 							  const PgFdwRelationInfo *fpinfo_o,
 							  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -558,6 +558,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1434,7 +1439,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2372,7 +2377,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2746,7 +2751,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3566,7 +3571,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4441,7 +4446,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4527,7 +4532,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4755,7 +4760,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index eef410db39..43ffd4f73f 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "libpq-fe.h"
@@ -129,7 +130,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -137,6 +138,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 							   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int	ExtractConnectionOptions(List *defelems,
@@ -203,6 +207,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 									bool is_subquery,
 									List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 83971665e3..ce5785c27a 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2628,9 +2651,98 @@ DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
 
--- Two-phase transactions are not supported.
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
 BEGIN;
-SELECT count(*) FROM ft1;
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
+INSERT INTO ft7_2pc VALUES(1);
 ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 94992be427..3f52daa11e 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -477,6 +477,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -504,6 +541,14 @@
    managed by creating corresponding remote savepoints.
   </para>
 
+  <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
   <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
-- 
2.21.1 (Apple Git-122.3)

commit e50a1deee6eefdfe4ac618db336ac850257f3c3f
Author: Muhammad Usama <m.usama@highgo.ca>
Date:   Fri Mon 27 13:14:10 2020 +0500

    minor fixes in transactions involving multiple servers patch

diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
index 0990a4e3ed..50e745b603 100644
--- a/src/backend/access/fdwxact/fdwxact.c
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -1543,7 +1543,7 @@ FdwXactResolveTransaction(Oid dbid, TransactionId xid, PGPROC *waiter)
 
 	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
 
-	while ((fdwxact = get_fdwxact_to_resolve(MyDatabaseId, xid)) != NULL)
+	while ((fdwxact = get_fdwxact_to_resolve(dbid, xid)) != NULL)
 	{
 		FdwXactRslvState *state;
 		ForeignServer *server;
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
index e293d13562..a871727661 100644
--- a/src/backend/access/fdwxact/launcher.c
+++ b/src/backend/access/fdwxact/launcher.c
@@ -394,7 +394,7 @@ fdwxact_relaunch_resolvers(void)
 	HASHCTL		ctl;
 	HASH_SEQ_STATUS status;
 	Oid		   *entry;
-	bool		launched;
+	bool		launched = false;
 	int			i;
 
 	memset(&ctl, 0, sizeof(ctl));
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 9dce03a6e4..26d6a08b14 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3856,6 +3856,7 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 			break;
 		case WAIT_EVENT_FDWXACT:
 			event_name = "FdwXact";
+			break;
 		case WAIT_EVENT_FDWXACT_RESOLUTION:
 			event_name = "FdwXactResolution";
 			break;

From c64479392f1d47d128d6c034697d251f6f37b527 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 21:41:29 +0500
Subject: [PATCH v20 5/5] Add regression tests for atomic commit.

---
 src/test/modules/Makefile                     |   1 +
 src/test/modules/test_fdwxact/.gitignore      |   4 +
 src/test/modules/test_fdwxact/Makefile        |  28 ++
 .../test_fdwxact/expected/test_fdwxact.out    | 188 +++++++
 src/test/modules/test_fdwxact/fdwxact.conf    |   7 +
 .../modules/test_fdwxact/sql/test_fdwxact.sql | 178 +++++++
 src/test/modules/test_fdwxact/t/001_basic.pl  | 119 +++++
 .../test_fdwxact/test_fdwxact--1.0.sql        |  44 ++
 src/test/modules/test_fdwxact/test_fdwxact.c  | 471 ++++++++++++++++++
 .../modules/test_fdwxact/test_fdwxact.control |   4 +
 src/test/recovery/Makefile                    |   2 +-
 src/test/recovery/t/021_fdwxact.pl            | 175 +++++++
 src/test/regress/pg_regress.c                 |  13 +-
 13 files changed, 1229 insertions(+), 5 deletions(-)
 create mode 100644 src/test/modules/test_fdwxact/.gitignore
 create mode 100644 src/test/modules/test_fdwxact/Makefile
 create mode 100644 src/test/modules/test_fdwxact/expected/test_fdwxact.out
 create mode 100644 src/test/modules/test_fdwxact/fdwxact.conf
 create mode 100644 src/test/modules/test_fdwxact/sql/test_fdwxact.sql
 create mode 100644 src/test/modules/test_fdwxact/t/001_basic.pl
 create mode 100644 src/test/modules/test_fdwxact/test_fdwxact--1.0.sql
 create mode 100644 src/test/modules/test_fdwxact/test_fdwxact.c
 create mode 100644 src/test/modules/test_fdwxact/test_fdwxact.control
 create mode 100644 src/test/recovery/t/021_fdwxact.pl

diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 29de73c060..8a48e6ba19 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -13,6 +13,7 @@ SUBDIRS = \
 		  test_bloomfilter \
 		  test_ddl_deparse \
 		  test_extensions \
+		  test_fdwxact \
 		  test_ginpostinglist \
 		  test_integerset \
 		  test_misc \
diff --git a/src/test/modules/test_fdwxact/.gitignore b/src/test/modules/test_fdwxact/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/src/test/modules/test_fdwxact/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/test_fdwxact/Makefile b/src/test/modules/test_fdwxact/Makefile
new file mode 100644
index 0000000000..b3fc99aee3
--- /dev/null
+++ b/src/test/modules/test_fdwxact/Makefile
@@ -0,0 +1,28 @@
+# src/test/modules/test_fdwxact/Makefile
+
+MODULE_big = test_fdwxact
+OBJS = \
+	$(WIN32RES) \
+	test_fdwxact.o
+PGFILEDESC = "test_fdwxact - test code for src/backend/access/fdwxact"
+
+EXTENSION = test_fdwxact
+DATA = test_fdwxact--1.0.sql
+
+REGRESS_OPTS = --temp-config=$(top_srcdir)/src/test/modules/test_fdwxact/fdwxact.conf
+REGRESS = test_fdwxact
+
+NO_INSTALLCHECK = 1
+
+TAP_TESTS =1
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_fdwxact/expected/test_fdwxact.out b/src/test/modules/test_fdwxact/expected/test_fdwxact.out
new file mode 100644
index 0000000000..a5c8b89655
--- /dev/null
+++ b/src/test/modules/test_fdwxact/expected/test_fdwxact.out
@@ -0,0 +1,188 @@
+--
+-- Test for foreign transaction management.
+--
+CREATE EXTENSION test_fdwxact;
+-- setup two servers that don't support transaction management API
+CREATE SERVER srv_1 FOREIGN DATA WRAPPER test_fdw;
+CREATE SERVER srv_2 FOREIGN DATA WRAPPER test_fdw;
+-- setup two servers that support only commit and rollback API
+CREATE SERVER srv_no2pc_1 FOREIGN DATA WRAPPER test_no2pc_fdw;
+CREATE SERVER srv_no2pc_2 FOREIGN DATA WRAPPER test_no2pc_fdw;
+-- setup two servers that support commit, rollback and prepare API.
+-- That is, those two server support two-phase commit protocol.
+CREATE SERVER srv_2pc_1 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE SERVER srv_2pc_2 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE TABLE t (i int);
+CREATE FOREIGN TABLE ft_1 (i int) SERVER srv_1;
+CREATE FOREIGN TABLE ft_2 (i int) SERVER srv_2;
+CREATE FOREIGN TABLE ft_no2pc_1 (i int) SERVER srv_no2pc_1;
+CREATE FOREIGN TABLE ft_no2pc_2 (i int) SERVER srv_no2pc_2;
+CREATE FOREIGN TABLE ft_2pc_1 (i int) SERVER srv_2pc_1;
+CREATE FOREIGN TABLE ft_2pc_2 (i int) SERVER srv_2pc_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_2;
+-- Test 'disabled' case.
+-- Modifies one or two servers but since we don't require two-phase
+-- commit, all case should not raise an error.
+SET foreign_twophase_commit TO disabled;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+ROLLBACK;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+ROLLBACK;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+ROLLBACK;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+-- Test 'required' case.
+-- In this case, when two-phase commit is required, all servers
+-- which are involved in the and modified need to support two-phase
+-- commit protocol. Otherwise transaction will rollback.
+SET foreign_twophase_commit TO 'required';
+-- Ok. Writing only one server doesn't require two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+-- Ok. Writing two servers, we require two-phase commit and success.
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+-- Ok. Only reading servers doesn't require two-phase commit.
+BEGIN;
+SELECT * FROM ft_2pc_1;
+ i 
+---
+(0 rows)
+
+SELECT * FROM ft_2pc_2;
+ i 
+---
+(0 rows)
+
+COMMIT;
+BEGIN;
+SELECT * FROM ft_1;
+ i 
+---
+(0 rows)
+
+SELECT * FROM ft_no2pc_1;
+ i 
+---
+(0 rows)
+
+COMMIT;
+-- Ok. Read one server and write one server.
+BEGIN;
+SELECT * FROM ft_1;
+ i 
+---
+(0 rows)
+
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+SELECT * FROM ft_no2pc_1;
+ i 
+---
+(0 rows)
+
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+-- Error. ft_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. ft_no2pc_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. Both ft_1 and ft_2 don't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. Both ft_no2pc_1 and ft_no2pc_2 don't support two-phase
+-- commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. Two-phase commit is required because of writes on two
+-- servers: local node and ft_no2pc_1. But ft_no2pc_1 doesn't support
+-- two-phase commit.
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Test 'prefer' case.
+-- The cases where failed in 'required' case shoul pass in 'prefer'.
+-- We simply commit/rollback a transaction in one-phase on a server
+-- that doesn't support two-phase commit, instead of error.
+SET foreign_twophase_commit TO 'prefer';
+-- We modify at least one server that doesn't support two-phase commit.
+-- These servers are committed in one-phase.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
diff --git a/src/test/modules/test_fdwxact/fdwxact.conf b/src/test/modules/test_fdwxact/fdwxact.conf
new file mode 100644
index 0000000000..20e4a671df
--- /dev/null
+++ b/src/test/modules/test_fdwxact/fdwxact.conf
@@ -0,0 +1,7 @@
+shared_preload_libraries = 'test_fdwxact'
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 1
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = disabled
diff --git a/src/test/modules/test_fdwxact/sql/test_fdwxact.sql b/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
new file mode 100644
index 0000000000..554312542f
--- /dev/null
+++ b/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
@@ -0,0 +1,178 @@
+--
+-- Test for foreign transaction management.
+--
+
+CREATE EXTENSION test_fdwxact;
+
+-- setup two servers that don't support transaction management API
+CREATE SERVER srv_1 FOREIGN DATA WRAPPER test_fdw;
+CREATE SERVER srv_2 FOREIGN DATA WRAPPER test_fdw;
+
+-- setup two servers that support only commit and rollback API
+CREATE SERVER srv_no2pc_1 FOREIGN DATA WRAPPER test_no2pc_fdw;
+CREATE SERVER srv_no2pc_2 FOREIGN DATA WRAPPER test_no2pc_fdw;
+
+-- setup two servers that support commit, rollback and prepare API.
+-- That is, those two server support two-phase commit protocol.
+CREATE SERVER srv_2pc_1 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE SERVER srv_2pc_2 FOREIGN DATA WRAPPER test_2pc_fdw;
+
+CREATE TABLE t (i int);
+CREATE FOREIGN TABLE ft_1 (i int) SERVER srv_1;
+CREATE FOREIGN TABLE ft_2 (i int) SERVER srv_2;
+CREATE FOREIGN TABLE ft_no2pc_1 (i int) SERVER srv_no2pc_1;
+CREATE FOREIGN TABLE ft_no2pc_2 (i int) SERVER srv_no2pc_2;
+CREATE FOREIGN TABLE ft_2pc_1 (i int) SERVER srv_2pc_1;
+CREATE FOREIGN TABLE ft_2pc_2 (i int) SERVER srv_2pc_2;
+
+CREATE USER MAPPING FOR PUBLIC SERVER srv_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_2;
+
+-- Test 'disabled' case.
+-- Modifies one or two servers but since we don't require two-phase
+-- commit, all case should not raise an error.
+SET foreign_twophase_commit TO disabled;
+
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+ROLLBACK;
+
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+ROLLBACK;
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+ROLLBACK;
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+
+-- Test 'required' case.
+-- In this case, when two-phase commit is required, all servers
+-- which are involved in the and modified need to support two-phase
+-- commit protocol. Otherwise transaction will rollback.
+SET foreign_twophase_commit TO 'required';
+
+-- Ok. Writing only one server doesn't require two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Ok. Writing two servers, we require two-phase commit and success.
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Ok. Only reading servers doesn't require two-phase commit.
+BEGIN;
+SELECT * FROM ft_2pc_1;
+SELECT * FROM ft_2pc_2;
+COMMIT;
+BEGIN;
+SELECT * FROM ft_1;
+SELECT * FROM ft_no2pc_1;
+COMMIT;
+
+-- Ok. Read one server and write one server.
+BEGIN;
+SELECT * FROM ft_1;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+SELECT * FROM ft_no2pc_1;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Error. ft_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Error. ft_no2pc_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Error. Both ft_1 and ft_2 don't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+
+-- Error. Both ft_no2pc_1 and ft_no2pc_2 don't support two-phase
+-- commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+
+-- Error. Two-phase commit is required because of writes on two
+-- servers: local node and ft_no2pc_1. But ft_no2pc_1 doesn't support
+-- two-phase commit.
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+
+-- Test 'prefer' case.
+-- The cases where failed in 'required' case shoul pass in 'prefer'.
+-- We simply commit/rollback a transaction in one-phase on a server
+-- that doesn't support two-phase commit, instead of error.
+SET foreign_twophase_commit TO 'prefer';
+
+-- We modify at least one server that doesn't support two-phase commit.
+-- These servers are committed in one-phase.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
diff --git a/src/test/modules/test_fdwxact/t/001_basic.pl b/src/test/modules/test_fdwxact/t/001_basic.pl
new file mode 100644
index 0000000000..c712fd1cf3
--- /dev/null
+++ b/src/test/modules/test_fdwxact/t/001_basic.pl
@@ -0,0 +1,119 @@
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 10;
+
+my $node = get_new_node('main');
+$node->init;
+$node->append_conf('postgresql.conf', qq(
+shared_preload_libraries = 'test_fdwxact'
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = required
+test_fdwxact.log_api_calls = true
+				   ));
+$node->start;
+
+$node->psql(
+	'postgres', "
+CREATE EXTENSION test_fdwxact;
+CREATE SERVER srv FOREIGN DATA WRAPPER test_fdw;
+CREATE SERVER srv_no2pc FOREIGN DATA WRAPPER test_no2pc_fdw;
+CREATE SERVER srv_2pc_1 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE SERVER srv_2pc_2 FOREIGN DATA WRAPPER test_2pc_fdw;
+
+CREATE TABLE t (i int);
+CREATE FOREIGN TABLE ft (i int) SERVER srv;
+CREATE FOREIGN TABLE ft_no2pc (i int) SERVER srv_no2pc;
+CREATE FOREIGN TABLE ft_2pc_1 (i int) SERVER srv_2pc_1;
+CREATE FOREIGN TABLE ft_2pc_2 (i int) SERVER srv_2pc_2;
+
+CREATE USER MAPPING FOR PUBLIC SERVER srv;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_2;
+	");
+
+sub run_transaction
+{
+	my ($node, $prepsql, $sql) = @_;
+
+	local $ENV{PGHOST} = $node->host;
+	local $ENV{PGPORT} = $node->port;
+
+	truncate $node->logfile, 0;
+
+	$node->safe_psql('postgres', $prepsql);
+	my ($cmdret, $stdout, $stderr) = $node->psql('postgres',
+												 "BEGIN;
+												 SELECT txid_current() as xid;
+												 $sql
+												 COMMIT;");
+	my $log = TestLib::slurp_file($node->logfile);
+
+	return $log, $stdout;
+}
+
+my ($log, $xid);
+
+# The transaction is committed using two-phase commit.
+($log, $xid) = run_transaction($node, "",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+like($log, qr/commit prepared tx_$xid on srv_2pc_1/, "commit prepared transaction-1");
+like($log, qr/commit prepared tx_$xid on srv_2pc_2/, "commit prepared transaction-2");
+
+# Similary, two-phase commit is used.
+($log, $xid) = run_transaction($node, "",
+					  "INSERT INTO t VALUES(1);
+					  INSERT INTO ft_2pc_1 VALUES(1);");
+like($log, qr/commit prepared tx_$xid on srv_2pc_1/, "commit prepared transaction-3");
+
+# Inject an error into prepare phase on srv_2pc_1. The transaction fails during
+# preparing the foreign transaction on srv_2pc_1. Then, we try to both 'rollback' and
+# 'rollback prepared' the foreign transaction, and rollback another foreign
+# transaction.
+($log, $xid) = run_transaction($node,
+							   "SELECT test_inject_error('error', 'prepare', 'srv_2pc_1');",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+like($log, qr/rollback $xid on srv_2pc_1/, "rollback on failed server");
+like($log, qr/rollback prepared tx_$xid on srv_2pc_1/, "rollback prepared on failed server");
+like($log, qr/rollback $xid on srv_2pc_2/, "rollback on another server");
+
+# Inject an panic into prepare phase on srv_2pc_2. The server crashes after preparing both
+# foreign transaction. After the restart, those transactions are recovered as in-doubt
+# transactions. We check if the resolver process rollbacks those transaction after recovery.
+($log, $xid) = run_transaction($node,
+							   "SELECT test_inject_error('panic', 'prepare', 'srv_2pc_2');",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+$node->restart();
+$node->poll_query_until('postgres',
+						"SELECT count(*) = 0 FROM pg_foreign_xacts")
+  or die "Timeout while waiting for resolver process to resolve in-doubt transactions";
+$log = TestLib::slurp_file($node->logfile);
+like($log, qr/rollback prepared tx_[0-9]+ on srv_2pc_1/, "resolver rolled back in-doubt transaction");
+like($log, qr/rollback prepared tx_[0-9]+ on srv_2pc_2/, "resolver rolled back in-doubt transaction");
+truncate $node->logfile, 0;
+
+# Inject an panic into commit phase on srv_2pc_1. The server crashes due to the panic
+# error raised by resolver process during commit prepared foreign transaction on srv_2pc_1.
+# After the restart, those transactions are recovered as in-doubt transactions. We check if
+# the resolver process commits those transaction after recovery.
+($log, $xid) = run_transaction($node,
+							   "SELECT test_inject_error('panic', 'commit', 'srv_2pc_1');",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+$node->restart();
+$node->poll_query_until('postgres',
+						"SELECT count(*) = 0 FROM pg_foreign_xacts")
+  or die "Timeout while waiting for resolver process to resolve in-doubt transactions";
+$log = TestLib::slurp_file($node->logfile);
+like($log, qr/commit prepared tx_[0-9]+ on srv_2pc_1/, "resolver rolled back in-doubt transaction");
+like($log, qr/commit prepared tx_[0-9]+ on srv_2pc_2/, "resolver rolled back in-doubt transaction");
+truncate $node->logfile, 0;
diff --git a/src/test/modules/test_fdwxact/test_fdwxact--1.0.sql b/src/test/modules/test_fdwxact/test_fdwxact--1.0.sql
new file mode 100644
index 0000000000..f676dfe04b
--- /dev/null
+++ b/src/test/modules/test_fdwxact/test_fdwxact--1.0.sql
@@ -0,0 +1,44 @@
+/* src/test/modules/test_atomic_commit/test_atomic_commit--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_atomic_commit" to load this file. \quit
+
+-- test_fdw doesn't use transaction API
+CREATE FUNCTION test_fdw_handler()
+RETURNS fdw_handler
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FOREIGN DATA WRAPPER test_fdw
+  HANDLER test_fdw_handler;
+
+-- test_no2pc_fdw uses only COMMIT and ROLLBACK API
+CREATE FUNCTION test_no2pc_fdw_handler()
+RETURNS fdw_handler
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FOREIGN DATA WRAPPER test_no2pc_fdw
+  HANDLER test_no2pc_fdw_handler;
+
+-- test_2pc uses PREPARE API as well
+CREATE FUNCTION test_2pc_fdw_handler()
+RETURNS fdw_handler
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FOREIGN DATA WRAPPER test_2pc_fdw
+  HANDLER test_2pc_fdw_handler;
+
+CREATE FUNCTION test_inject_error(
+elevel text,
+phase text,
+server text)
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION test_reset_error()
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
diff --git a/src/test/modules/test_fdwxact/test_fdwxact.c b/src/test/modules/test_fdwxact/test_fdwxact.c
new file mode 100644
index 0000000000..d8bfe48d96
--- /dev/null
+++ b/src/test/modules/test_fdwxact/test_fdwxact.c
@@ -0,0 +1,471 @@
+/*-------------------------------------------------------------------------
+ *
+ * test_fdwxact.c
+ *		  Test modules for foreign transaction management
+ *
+ * This module implements three types of foreign data wrapper: the first
+ * doesn't support any transaction FDW APIs, the second supports only
+ * commit and rollback API and the third supports all transaction API including
+ * prepare.
+ *
+ * Also, this module has an ability to inject an error at prepare callback or
+ * commit callback using test_inject_error() SQL function. The information of
+ * injected error is stored in the shared memory so that backend processes and
+ * resolver processes can see it.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		src/test/modules/test_fdwxact/test_fdwxact.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact.h"
+#include "commands/defrem.h"
+#include "access/reloptions.h"
+#include "foreign/fdwapi.h"
+#include "funcapi.h"
+#include "miscadmin.h"
+#include "optimizer/clauses.h"
+#include "optimizer/cost.h"
+#include "optimizer/optimizer.h"
+#include "optimizer/pathnode.h"
+#include "optimizer/paths.h"
+#include "optimizer/planmain.h"
+#include "optimizer/restrictinfo.h"
+#include "storage/ipc.h"
+#include "storage/spin.h"
+#include "utils/builtins.h"
+
+PG_MODULE_MAGIC;
+
+#define TEST_FDWXCT_MAX_NAME_LEN 32
+
+typedef struct testFdwXactSharedState
+{
+	char	elevel[TEST_FDWXCT_MAX_NAME_LEN];
+	char	phase[TEST_FDWXCT_MAX_NAME_LEN];
+	char	server[TEST_FDWXCT_MAX_NAME_LEN];
+	LWLock	*lock;
+} testFdwXactSharedState;
+testFdwXactSharedState *fxss = NULL;
+
+static shmem_startup_hook_type prev_shmem_startup_hook = NULL;
+static bool log_api_calls = false;
+
+void _PG_init(void);
+void _PG_fini(void);
+PG_FUNCTION_INFO_V1(test_fdw_handler);
+PG_FUNCTION_INFO_V1(test_no2pc_fdw_handler);
+PG_FUNCTION_INFO_V1(test_2pc_fdw_handler);
+PG_FUNCTION_INFO_V1(test_inject_error);
+PG_FUNCTION_INFO_V1(test_reset_error);
+
+static void test_fdwxact_shmem_startup(void);
+static bool check_event(char *servername, char *phase, int *elevel);
+static void testGetForeignRelSize(PlannerInfo *root,
+								  RelOptInfo *baserel,
+								  Oid foreigntableid);
+static void testGetForeignPaths(PlannerInfo *root,
+								RelOptInfo *baserel,
+								Oid foreigntableid);
+static ForeignScan *testGetForeignPlan(PlannerInfo *root,
+									   RelOptInfo *foreignrel,
+									   Oid foreigntableid,
+									   ForeignPath *best_path,
+									   List *tlist,
+									   List *scan_clauses,
+									   Plan *outer_plan);
+static void testBeginForeignScan(ForeignScanState *node, int eflags);
+static TupleTableSlot *testIterateForeignScan(ForeignScanState *node);
+static void testReScanForeignScan(ForeignScanState *node);
+static void testEndForeignScan(ForeignScanState *node);
+static void testBeginForeignModify(ModifyTableState *mtstate,
+								   ResultRelInfo *resultRelInfo,
+								   List *fdw_private,
+								   int subplan_index,
+								   int eflags);
+static TupleTableSlot *testExecForeignInsert(EState *estate,
+											 ResultRelInfo *resultRelInfo,
+											 TupleTableSlot *slot,
+											 TupleTableSlot *planSlot);
+static void testEndForeignModify(EState *estate,
+								 ResultRelInfo *resultRelInfo);
+static void testBeginForeignInsert(ModifyTableState *mtstate,
+								   ResultRelInfo *resultRelInfo);
+static void testEndForeignInsert(EState *estate,
+								 ResultRelInfo *resultRelInfo);
+static int	testIsForeignRelUpdatable(Relation rel);
+static void testPrepareForeignTransaction(FdwXactRslvState *state);
+static void testCommitForeignTransaction(FdwXactRslvState *state);
+static void testRollbackForeignTransaction(FdwXactRslvState *state);
+static char *testGetPrepareId(TransactionId xid, Oid serverid,
+							  Oid userid, int *prep_id_len);
+
+void
+_PG_init(void)
+{
+	DefineCustomBoolVariable("test_fdwxact.log_api_calls",
+							 "Report transaction API calls to logs.",
+							 NULL,
+							 &log_api_calls,
+							 false,
+							 PGC_USERSET,
+							 0,
+							 NULL, NULL, NULL);
+
+	RequestAddinShmemSpace(MAXALIGN(sizeof(testFdwXactSharedState)));
+	RequestNamedLWLockTranche("test_fdwxact", 1);
+
+	prev_shmem_startup_hook = shmem_startup_hook;
+	shmem_startup_hook = test_fdwxact_shmem_startup;
+}
+
+void
+_PG_fini(void)
+{
+	/* Uninstall hooks. */
+	shmem_startup_hook = prev_shmem_startup_hook;
+}
+
+static void
+test_fdwxact_shmem_startup(void)
+{
+	bool found;
+
+	if (prev_shmem_startup_hook)
+		prev_shmem_startup_hook();
+
+	fxss = ShmemInitStruct("test_fdwxact",
+						   sizeof(testFdwXactSharedState),
+						   &found);
+	if (!found)
+	{
+		memset(fxss->elevel, 0, TEST_FDWXCT_MAX_NAME_LEN);
+		memset(fxss->phase, 0, TEST_FDWXCT_MAX_NAME_LEN);
+		memset(fxss->server, 0, TEST_FDWXCT_MAX_NAME_LEN);
+		fxss->lock = &(GetNamedLWLockTranche("test_fdwxact"))->lock;
+	}
+}
+
+Datum
+test_fdw_handler(PG_FUNCTION_ARGS)
+{
+	FdwRoutine *routine = makeNode(FdwRoutine);
+
+	/* Functions for scanning foreign tables */
+	routine->GetForeignRelSize = testGetForeignRelSize;
+	routine->GetForeignPaths = testGetForeignPaths;
+	routine->GetForeignPlan = testGetForeignPlan;
+	routine->BeginForeignScan = testBeginForeignScan;
+	routine->IterateForeignScan = testIterateForeignScan;
+	routine->ReScanForeignScan = testReScanForeignScan;
+	routine->EndForeignScan = testEndForeignScan;
+
+	/* Functions for updating foreign tables */
+	routine->AddForeignUpdateTargets = NULL;
+	routine->PlanForeignModify = NULL;
+	routine->BeginForeignModify = testBeginForeignModify;
+	routine->ExecForeignInsert = testExecForeignInsert;
+	routine->EndForeignModify = testEndForeignModify;
+	routine->BeginForeignInsert = testBeginForeignInsert;
+	routine->EndForeignInsert = testEndForeignInsert;
+	routine->IsForeignRelUpdatable = testIsForeignRelUpdatable;
+
+	PG_RETURN_POINTER(routine);
+}
+
+Datum
+test_no2pc_fdw_handler(PG_FUNCTION_ARGS)
+{
+	FdwRoutine *routine = makeNode(FdwRoutine);
+
+	/* Functions for scanning foreign tables */
+	routine->GetForeignRelSize = testGetForeignRelSize;
+	routine->GetForeignPaths = testGetForeignPaths;
+	routine->GetForeignPlan = testGetForeignPlan;
+	routine->BeginForeignScan = testBeginForeignScan;
+	routine->IterateForeignScan = testIterateForeignScan;
+	routine->ReScanForeignScan = testReScanForeignScan;
+	routine->EndForeignScan = testEndForeignScan;
+
+	/* Functions for updating foreign tables */
+	routine->AddForeignUpdateTargets = NULL;
+	routine->PlanForeignModify = NULL;
+	routine->BeginForeignModify = testBeginForeignModify;
+	routine->ExecForeignInsert = testExecForeignInsert;
+	routine->EndForeignModify = testEndForeignModify;
+	routine->BeginForeignInsert = testBeginForeignInsert;
+	routine->EndForeignInsert = testEndForeignInsert;
+	routine->IsForeignRelUpdatable = testIsForeignRelUpdatable;
+
+	/* Support only COMMIT and ROLLBACK */
+	routine->CommitForeignTransaction = testCommitForeignTransaction;
+	routine->RollbackForeignTransaction = testRollbackForeignTransaction;
+
+	PG_RETURN_POINTER(routine);
+}
+
+Datum
+test_2pc_fdw_handler(PG_FUNCTION_ARGS)
+{
+	FdwRoutine *routine = makeNode(FdwRoutine);
+
+	/* Functions for scanning foreign tables */
+	routine->GetForeignRelSize = testGetForeignRelSize;
+	routine->GetForeignPaths = testGetForeignPaths;
+	routine->GetForeignPlan = testGetForeignPlan;
+	routine->BeginForeignScan = testBeginForeignScan;
+	routine->IterateForeignScan = testIterateForeignScan;
+	routine->ReScanForeignScan = testReScanForeignScan;
+	routine->EndForeignScan = testEndForeignScan;
+
+	/* Functions for updating foreign tables */
+	routine->AddForeignUpdateTargets = NULL;
+	routine->PlanForeignModify = NULL;
+	routine->BeginForeignModify = testBeginForeignModify;
+	routine->ExecForeignInsert = testExecForeignInsert;
+	routine->EndForeignModify = testEndForeignModify;
+	routine->BeginForeignInsert = testBeginForeignInsert;
+	routine->EndForeignInsert = testEndForeignInsert;
+	routine->IsForeignRelUpdatable = testIsForeignRelUpdatable;
+
+	/* Support all functions for foreign transactions */
+	routine->GetPrepareId = testGetPrepareId;
+	routine->PrepareForeignTransaction = testPrepareForeignTransaction;
+	routine->CommitForeignTransaction = testCommitForeignTransaction;
+	routine->RollbackForeignTransaction = testRollbackForeignTransaction;
+
+	PG_RETURN_POINTER(routine);
+}
+
+static void
+testGetForeignRelSize(PlannerInfo *root,
+					  RelOptInfo *baserel,
+					  Oid foreigntableid)
+{
+	baserel->pages = 10;
+	baserel->tuples = 100;
+}
+
+static void
+testGetForeignPaths(PlannerInfo *root,
+					RelOptInfo *baserel,
+					Oid foreigntableid)
+{
+	add_path(baserel, (Path *) create_foreignscan_path(root, baserel,
+													   NULL,
+													   10, 10, 10,
+													   NIL,
+													   baserel->lateral_relids,
+													   NULL, NIL));
+}
+
+static ForeignScan *
+testGetForeignPlan(PlannerInfo *root,
+				   RelOptInfo *foreignrel,
+				   Oid foreigntableid,
+				   ForeignPath *best_path,
+				   List *tlist,
+				   List *scan_clauses,
+				   Plan *outer_plan)
+{
+	return make_foreignscan(tlist,
+							NIL,
+							foreignrel->relid,
+							NIL,
+							NULL,
+							NIL,
+							NIL,
+							outer_plan);
+}
+
+static void
+testBeginForeignScan(ForeignScanState *node, int eflags)
+{
+	return;
+}
+
+static TupleTableSlot *
+testIterateForeignScan(ForeignScanState *node)
+{
+	return ExecClearTuple(node->ss.ss_ScanTupleSlot);
+}
+
+static void
+testReScanForeignScan(ForeignScanState *node)
+{
+	return;
+}
+
+static void
+testEndForeignScan(ForeignScanState *node)
+{
+	return;
+}
+
+static void
+testBeginForeignModify(ModifyTableState *mtstate,
+					   ResultRelInfo *resultRelInfo,
+					   List *fdw_private,
+					   int subplan_index,
+					   int eflags)
+{
+	return;
+}
+
+static TupleTableSlot *
+testExecForeignInsert(EState *estate,
+					  ResultRelInfo *resultRelInfo,
+					  TupleTableSlot *slot,
+					  TupleTableSlot *planSlot)
+{
+	return slot;
+}
+
+static void
+testEndForeignModify(EState *estate,
+					 ResultRelInfo *resultRelInfo)
+{
+	return;
+}
+
+static void
+testBeginForeignInsert(ModifyTableState *mtstate,
+					   ResultRelInfo *resultRelInfo)
+{
+	return;
+}
+
+static void
+testEndForeignInsert(EState *estate,
+					 ResultRelInfo *resultRelInfo)
+{
+	return;
+}
+
+static int
+testIsForeignRelUpdatable(Relation rel)
+{
+	/* allow only inserts */
+	return (1 << CMD_INSERT);
+}
+
+static char *
+testGetPrepareId(TransactionId xid, Oid serverid,
+				 Oid userid, int *prep_id_len)
+{
+	static char buf[32] = {0};
+
+	*prep_id_len = snprintf(buf, 32, "tx_%u", xid);
+
+	return buf;
+}
+
+static void
+testPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	int elevel;
+
+	if (check_event(state->server->servername, "prepare", &elevel))
+		elog(elevel, "injected error at prepare");
+
+	if (log_api_calls)
+		ereport(LOG, (errmsg("prepare %s on %s",
+							 state->fdwxact_id,
+							 state->server->servername)));
+}
+
+static void
+testCommitForeignTransaction(FdwXactRslvState *state)
+{
+	int elevel;
+
+	if (check_event(state->server->servername, "commit", &elevel))
+		elog(elevel, "injected error at commit");
+
+	if (log_api_calls)
+	{
+		if (state->flags && FDWXACT_FLAG_ONEPHASE)
+			ereport(LOG, (errmsg("commit %u on %s",
+								 state->xid, state->server->servername)));
+		else
+			ereport(LOG, (errmsg("commit prepared %s on %s",
+								 state->fdwxact_id,
+								 state->server->servername)));
+	}
+}
+
+static void
+testRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	if (log_api_calls)
+	{
+		if (state->flags && FDWXACT_FLAG_ONEPHASE)
+			ereport(LOG, (errmsg("rollback %u on %s",
+								 state->xid, state->server->servername)));
+		else
+			ereport(LOG, (errmsg("rollback prepared %s on %s",
+								 state->fdwxact_id,
+								 state->server->servername)));
+	}
+}
+
+/*
+ * Check if an event is set at the phase on the server. If there is, set
+ * elevel and return true.
+ */
+static bool
+check_event(char *servername, char *phase, int *elevel)
+{
+	LWLockAcquire(fxss->lock, LW_SHARED);
+
+	if (strcasecmp(fxss->server, servername) != 0 ||
+		strcasecmp(fxss->phase, phase) != 0)
+	{
+		LWLockRelease(fxss->lock);
+		return false;
+	}
+
+	/* Currently support only error and panic */
+	if (strcasecmp(fxss->elevel, "error") == 0)
+		*elevel = ERROR;
+	if (strcasecmp(fxss->elevel, "panic") == 0)
+		*elevel = PANIC;
+
+	LWLockRelease(fxss->lock);
+
+	return true;
+}
+
+/* SQL function to inject an error */
+Datum
+test_inject_error(PG_FUNCTION_ARGS)
+{
+	char *elevel = text_to_cstring(PG_GETARG_TEXT_P(0));
+	char *phase = text_to_cstring(PG_GETARG_TEXT_P(1));
+	char *server = text_to_cstring(PG_GETARG_TEXT_P(2));
+
+	LWLockAcquire(fxss->lock, LW_EXCLUSIVE);
+	strncpy(fxss->elevel, elevel, TEST_FDWXCT_MAX_NAME_LEN);
+	strncpy(fxss->phase, phase, TEST_FDWXCT_MAX_NAME_LEN);
+	strncpy(fxss->server, server, TEST_FDWXCT_MAX_NAME_LEN);
+	LWLockRelease(fxss->lock);
+
+	PG_RETURN_NULL();
+}
+
+/* SQL function to reset an error */
+Datum
+test_reset_error(PG_FUNCTION_ARGS)
+{
+	LWLockAcquire(fxss->lock, LW_EXCLUSIVE);
+	memset(fxss->elevel, 0, TEST_FDWXCT_MAX_NAME_LEN);
+	memset(fxss->phase, 0, TEST_FDWXCT_MAX_NAME_LEN);
+	memset(fxss->server, 0, TEST_FDWXCT_MAX_NAME_LEN);
+	LWLockRelease(fxss->lock);
+
+	PG_RETURN_NULL();
+}
diff --git a/src/test/modules/test_fdwxact/test_fdwxact.control b/src/test/modules/test_fdwxact/test_fdwxact.control
new file mode 100644
index 0000000000..ac9945ba03
--- /dev/null
+++ b/src/test/modules/test_fdwxact/test_fdwxact.control
@@ -0,0 +1,4 @@
+comment = 'Test code for fdwxact'
+default_version = '1.0'
+module_pathname = '$libdir/test_fdwxact'
+relocatable = true
diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index fa8e031526..d47d96975b 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/021_fdwxact.pl b/src/test/recovery/t/021_fdwxact.pl
new file mode 100644
index 0000000000..9af9bb81dc
--- /dev/null
+++ b/src/test/recovery/t/021_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 38b2b1e8e1..f30fe6b492 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2335,9 +2335,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2352,7 +2355,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m %b[%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.23.0

From 028f5416a4382821bb36d00cc374d54bbd9ab02b Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 21:27:18 +0500
Subject: [PATCH v20 3/5] Documentation update.

---
 doc/src/sgml/catalogs.sgml                | 145 +++++++++++++
 doc/src/sgml/config.sgml                  | 147 +++++++++++++-
 doc/src/sgml/distributed-transaction.sgml | 154 ++++++++++++++
 doc/src/sgml/fdwhandler.sgml              | 236 ++++++++++++++++++++++
 doc/src/sgml/filelist.sgml                |   1 +
 doc/src/sgml/func.sgml                    |  89 ++++++++
 doc/src/sgml/monitoring.sgml              |  60 ++++++
 doc/src/sgml/postgres.sgml                |   1 +
 doc/src/sgml/storage.sgml                 |   6 +
 9 files changed, 838 insertions(+), 1 deletion(-)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index fcadba331d..8daeb7c006 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8237,6 +8237,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
       <entry>open cursors</entry>
      </row>
 
+     <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
      <row>
       <entry><link linkend="view-pg-file-settings"><structname>pg_file_settings</structname></link></entry>
       <entry>summary of configuration file contents</entry>
@@ -9695,6 +9700,146 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with that this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on that the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>initial</literal> : Initial status.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>resolved</literal> : This foreign transaction has been resolved.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-doubt status and
+       needs to be resolved by calling <function>pg_resolve_fdwxact</function>
+       function.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 9f2a4a2470..e835c19dac 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -4487,7 +4487,6 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
 
      </variablelist>
     </sect2>
-
    </sect1>
 
    <sect1 id="runtime-config-query">
@@ -9081,6 +9080,152 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether distributed transaction commits ensures that all
+         involved changes on foreign servers are committed or not. Valid
+         values are <literal>required</literal>, <literal>prefer</literal> and
+         <literal>disabled</literal>. The default setting is
+         <literal>disabled</literal>. Setting to <literal>disabled</literal>
+         don't use two-phase commit protocol to commit or rollback distributed
+         transactions. When set to <literal>required</literal> distributed
+         transactions strictly requires that all written servers can use
+         two-phase commit protocol.  That is, the distributed transaction cannot
+         commit if even one server does not support the prepare callback routine
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. In <literal>prefer</literal> and <literal>required</literal> case,
+         distributed transaction commit will wait for all involving foreign
+         transaction to be committed before the command return a "success"
+         indication to the client.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+
+        <note>
+         <para>
+          When <literal>disabled</literal> or <literal>prefer</literal> there
+          can be risk of database consistency among all servers that involved in
+          the distributed transaction when some foreign server crashes during
+          committing the distributed transaction.
+         </para>
+        </note>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000000..85d8e8e9e4
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,154 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Formerly, transactions on foreign server were not simply committed or rolled
+   back one by one. Therefore, when one foreign server had a problem during
+   commit, it was possible that transactions on only part of foreign servers
+   are committed while other transactions are rolled back. This used to leave
+   database data in an inconsistent state in term of federated database.
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatically,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).
+    A <productname>PostgreSQL</productname> server that received SQL is called
+    <firstterm>coordinator node</firstterm> who is responsible for coordinating
+    all the participanting transactions. Using two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed
+    transaction manager prepares all transaction on the foreign servers if
+    two-phase commit is required. Two-phase commit is required when the
+    transaction modifies data on two or more servers including the local server
+    itself and <xref linkend="guc-foreign-twophase-commit"/>is
+    <literal>required</literal> or <literal>prefer</literal>. If all preparations
+    on foreign servers got successful go to the next step. Any failure happens
+    in this step <productname>PostgreSQL</productname> changes to rollback, then
+    rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the
+    transaction locally. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes rollback, then rollback all
+    transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>Manual Resolution of In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, distributed transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the coordinator node crashed during either preparing or
+    resolving distributed transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="pg-stat-foreign-xact-view"/>
+    view. These foreign transactions are resolved by foreign transaction resolver
+    process or executing <function>pg_resolve_foriegn_xact</function> function
+    manually.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolution">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that is
+    responsible for resolving both foreign transactions that are prepared by
+    online transactions and in-doubt transactions. They commit or rollback
+    prepared transaction on foreign servers if the coordinator received agreement
+    messages from all foreign servers during the first step.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on one database connecting to. On failure during resolution, they retry to
+    resolve at an interval of <varname>foreign_transaction_resolution_interval</varname>
+    time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped. So to drop the database, you can call
+     <function>pg_stop_foreign_xact_resovler</function> function to stop the
+     particular resolver process before dropping the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 6587678af2..dd0358ef22 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1415,6 +1415,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1894,4 +2015,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 68179f71cd..1ab8e80fdc 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 7c06afd3ea..e281bd33d8 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26126,6 +26126,95 @@ SELECT convert_from(pg_read_binary_file('file_in_utf8.txt'), 'UTF8');
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 579ccd34d4..9073c01eb9 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -384,6 +384,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_foreign_xact</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1271,6 +1279,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry><literal>CheckpointerMain</literal></entry>
          <entry>Waiting in main loop of checkpointer process.</entry>
         </row>
+        <row>
+         <entry><literal>FdwXactLauncherMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+        </row>
+        <row>
+         <entry><literal>FdwXactResolverMain</literal></entry>
+         <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+        </row>
+        <row>
+         <entry><literal>LogicalLauncherMain</literal></entry>
+         <entry>Waiting in main loop of logical launcher process.</entry>
+        </row>
         <row>
          <entry><literal>LogicalApplyMain</literal></entry>
          <entry>Waiting in main loop of logical apply process.</entry>
@@ -1506,6 +1526,10 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
          <entry><literal>SafeSnapshot</literal></entry>
          <entry>Waiting for a snapshot for a <literal>READ ONLY DEFERRABLE</literal> transaction.</entry>
         </row>
+        <row>
+         <entry><literal>FdwXactResolution</literal></entry>
+         <entry>Waiting for all foreign transaction participants to be resolved during atomic commit among foreign servers.</entry>
+        </row>
         <row>
          <entry><literal>SyncRep</literal></entry>
          <entry>Waiting for confirmation from remote server during synchronous replication.</entry>
@@ -2430,6 +2454,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-view" xreflabel="pg_stat_foreign_xact">
+   <title><structname>pg_stat_foreign_xact</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index c41ce9499b..5ef1f4a329 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -170,6 +170,7 @@ break is not needed in a wider output rendering.
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index 3234adb639..83f30c5045 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -83,6 +83,12 @@ Item
   subsystem</entry>
 </row>
 
+<row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
 <row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
-- 
2.23.0

From 76b4e2bc7977c1338d7fe72f4b27cd03ea42a0c4 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 14:12:17 +0500
Subject: [PATCH v20 1/5] Keep track of writing on non-temporary relation

---
 src/backend/executor/nodeModifyTable.c | 16 ++++++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 22 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 20a4c474cc..1ec07bad07 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -581,6 +581,10 @@ ExecInsert(ModifyTableState *mtstate,
 										   NULL,
 										   specToken);
 
+			/* Make note that we've wrote on non-temporary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -619,6 +623,10 @@ ExecInsert(ModifyTableState *mtstate,
 							   estate->es_output_cid,
 							   0, NULL);
 
+			/* Make note that we've wrote on non-temporary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -970,6 +978,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temporary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1482,6 +1494,10 @@ lreplace:;
 	if (canSetTag)
 		(estate->es_processed)++;
 
+	/* Make note that we've wrote on non-temporary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/* AFTER ROW UPDATE Triggers */
 	ExecARUpdateTriggers(estate, resultRelInfo, tupleid, oldtuple, slot,
 						 recheckIndexes,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7ee04babc2..a04fc70326 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -102,6 +102,12 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
-- 
2.23.0

From 910d440283c4582ff2706c37661a08ab036057b9 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 21:28:58 +0500
Subject: [PATCH v20 4/5] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/Makefile                 |   7 +-
 contrib/postgres_fdw/connection.c             | 603 +++++++++++-------
 .../postgres_fdw/expected/postgres_fdw.out    | 280 +++++++-
 contrib/postgres_fdw/fdwxact.conf             |   3 +
 contrib/postgres_fdw/postgres_fdw.c           |  21 +-
 contrib/postgres_fdw/postgres_fdw.h           |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql     | 124 +++-
 doc/src/sgml/postgres-fdw.sgml                |  45 ++
 8 files changed, 831 insertions(+), 259 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index ee8a80a392..91fa6e39fc 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -16,7 +16,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -29,3 +29,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index e45647f3ea..bb859e2927 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2020, PostgreSQL Global Development Group
  *
@@ -12,6 +12,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "catalog/pg_user_mapping.h"
@@ -56,6 +57,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -69,17 +71,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 								   SubTransactionId mySubid,
 								   SubTransactionId parentSubid,
@@ -92,23 +90,26 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 									 PGresult **result);
 static bool UserMappingPasswordRequired(UserMapping *user);
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +129,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,12 +136,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -155,6 +149,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -182,6 +191,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +200,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +211,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,11 +227,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -472,7 +520,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -699,193 +747,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 	PG_END_TRY();
 }
 
-/*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow any remote transactions, since it's not
-					 * very reasonable to hold them open until the prepared
-					 * transaction is committed.  For the moment, throw error
-					 * unconditionally; later we might allow read-only cases.
-					 * Note that the error will cause us to come right back
-					 * here with event == XACT_EVENT_ABORT, so we'll clean up
-					 * the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot PREPARE a transaction that has operated on postgres_fdw foreign tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
 /*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
@@ -902,10 +763,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -916,6 +773,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1250,3 +1111,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recursively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 90db550b92..8c31e26406 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -191,15 +210,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8923,10 +8944,10 @@ RESET ROLE;
 ALTER USER MAPPING FOR regress_nosuper SERVER loopback_nopw OPTIONS (ADD password_required 'false');
 SET ROLE regress_nosuper;
 -- Should finally work now
-SELECT * FROM ft1_nopw LIMIT 1;
-  c1  | c2 | c3 | c4 | c5 | c6 |     c7     | c8 
-------+----+----+----+----+----+------------+----
- 1111 |  2 |    |    |    |    | ft1        | 
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
+ c1 | c2 |        c3         |              c4              |            c5            | c6 |     c7     | c8  
+----+----+-------------------+------------------------------+--------------------------+----+------------+-----
+  1 |  2 | 00001_trig_update | Fri Jan 02 00:00:00 1970 PST | Fri Jan 02 00:00:00 1970 | 1  | 1          | foo
 (1 row)
 
 -- unpriv user also cannot set sslcert / sslkey on the user mapping
@@ -8943,16 +8964,16 @@ HINT:  User mappings with the sslcert or sslkey options set may only be created
 DROP USER MAPPING FOR CURRENT_USER SERVER loopback_nopw;
 -- This will fail again as it'll resolve the user mapping for public, which
 -- lacks password_required=false
-SELECT * FROM ft1_nopw LIMIT 1;
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
 ERROR:  password is required
 DETAIL:  Non-superusers must provide a password in the user mapping.
 RESET ROLE;
 -- The user mapping for public is passwordless and lacks the password_required=false
 -- mapping option, but will work because the current user is a superuser.
 SELECT * FROM ft1_nopw LIMIT 1;
-  c1  | c2 | c3 | c4 | c5 | c6 |     c7     | c8 
-------+----+----+----+----+----+------------+----
- 1111 |  2 |    |    |    |    | ft1        | 
+ c1 | c2 |  c3   |              c4              |            c5            | c6 |     c7     | c8  
+----+----+-------+------------------------------+--------------------------+----+------------+-----
+  6 |  6 | 00006 | Wed Jan 07 00:00:00 1970 PST | Wed Jan 07 00:00:00 1970 | 6  | 6          | foo
 (1 row)
 
 -- cleanup
@@ -8961,16 +8982,225 @@ DROP OWNED BY regress_nosuper;
 DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
--- Two-phase transactions are not supported.
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
 BEGIN;
-SELECT count(*) FROM ft1;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" of relation "T 5" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
  count 
 -------
-   822
+     0
 (1 row)
 
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
-ERROR:  cannot PREPARE a transaction that has operated on postgres_fdw foreign tables
-ROLLBACK;
-WARNING:  there is no transaction in progress
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000000..3fdbf93cdb
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 9fc53cad68..105451d199 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include <limits.h>
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -504,7 +505,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 							  const PgFdwRelationInfo *fpinfo_o,
 							  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -558,6 +558,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1434,7 +1439,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2372,7 +2377,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2746,7 +2751,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3566,7 +3571,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4441,7 +4446,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4527,7 +4532,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4755,7 +4760,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index eef410db39..43ffd4f73f 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "libpq-fe.h"
@@ -129,7 +130,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -137,6 +138,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 							   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int	ExtractConnectionOptions(List *defelems,
@@ -203,6 +207,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 									bool is_subquery,
 									List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 83971665e3..1ef66123df 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2598,7 +2621,7 @@ ALTER USER MAPPING FOR regress_nosuper SERVER loopback_nopw OPTIONS (ADD passwor
 SET ROLE regress_nosuper;
 
 -- Should finally work now
-SELECT * FROM ft1_nopw LIMIT 1;
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
 
 -- unpriv user also cannot set sslcert / sslkey on the user mapping
 -- first set password_required so we see the right error messages
@@ -2612,7 +2635,7 @@ DROP USER MAPPING FOR CURRENT_USER SERVER loopback_nopw;
 
 -- This will fail again as it'll resolve the user mapping for public, which
 -- lacks password_required=false
-SELECT * FROM ft1_nopw LIMIT 1;
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
 
 RESET ROLE;
 
@@ -2628,9 +2651,98 @@ DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
 
--- Two-phase transactions are not supported.
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
 BEGIN;
-SELECT count(*) FROM ft1;
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
 ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index 94992be427..3f52daa11e 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -477,6 +477,43 @@
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -504,6 +541,14 @@
    managed by creating corresponding remote savepoints.
   </para>
 
+  <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
   <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
-- 
2.23.0

From 811ef5995e30b4047f951886e3a8aa64ecf9469d Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 21:22:17 +0500
Subject: [PATCH v20 2/5] Support atomic commit among multiple foreign servers.

---
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  110 +
 src/backend/access/fdwxact/fdwxact.c          | 2736 +++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  560 ++++
 src/backend/access/fdwxact/resolver.c         |  436 +++
 src/backend/access/rmgrdesc/Makefile          |    1 +
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   58 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   56 +
 src/backend/access/transam/xact.c             |   26 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |    3 +
 src/backend/commands/copy.c                   |    6 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    8 +
 src/backend/executor/nodeForeignscan.c        |   24 +
 src/backend/executor/nodeModifyTable.c        |    6 +
 src/backend/foreign/foreign.c                 |   55 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   20 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   82 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/fdwxactdesc.c              |    1 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  162 +
 src/include/access/fdwxact_launcher.h         |   28 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   54 +
 src/include/access/resolver_internal.h        |   63 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   22 +
 src/include/foreign/fdwapi.h                  |   12 +
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    9 +-
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    3 +
 src/test/regress/expected/rules.out           |    7 +
 55 files changed, 4796 insertions(+), 18 deletions(-)
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100644 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 120000 src/bin/pg_waldump/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a8bb..49480dd039 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000000..0207a66fb4
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000000..361a46f8e3
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,110 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a consistent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+---------------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consists
+of the following four steps:
+
+1. Foreign Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant. The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+We record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE each foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared transaction on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step (commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itself to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rollbacked the resolver process wake up the waiter.
+
+
+Foreign Data Wrapper Callbacks for Transaction Management
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForeignTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for PREPARE, COMMIT or ROLLBACK
+the transaction on the foreign server, respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transaction in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions will have an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_PREPARING
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING before the foreign
+transaction is committed and aborted by FDW callback functions respectively.
+FdwXact entry is removed once the foreign transaction is resolved with WAL
+logging.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*1). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                     PREPARING                      |----+
+ +----------------------------------------------------+    |
+                          |                                |
+	                  v                                |
+ +----------------------------------------------------+    |
+ |                    PREPARED(*1)                    |    | (*2)
+ +----------------------------------------------------+    |
+           |                               |               |
+           v                               v               |
+ +--------------------+          +--------------------+    |
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |<---+
+ +--------------------+          +--------------------+
+
+(*1) Recovered FdwXact entries starts with PREPARED
+(*2) Paths when an error occurrs during preparing
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100644
index 0000000000..fdc6b1f415
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2736 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers atomically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * Two-phase commit protocol is used when the transaction modified two or
+ * more servers including the local node.  If two-phase commit protocol
+ * is not required all foreign transactions are committed at pre-commit
+ * phase.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit.  The foreign servers are
+ * registered if FDW has both CommitForeignTransaction API and
+ * RollbackForeignTransaction API.  Registered participant servers are
+ * identified by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * all foreign servers.  And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask to commit, we also tell to notify us when
+ * it's done, so that we can wait interruptibly to finish, and so that
+ * we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request.  We have one queue
+ * of which elements are ordered by the timestamp when they expect to be
+ * processed.  Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp when they expects to be processed.
+ * On failure, it enqueues again with new timestamp (last timestamp +
+ * foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction).  Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by
+ * pg_resovle_foreign_xact() function.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed, the corresponding FdwXact
+ * entry is update.  To avoid holding the lock during transaction processing
+ * which may take an unpredicatable time the in-memory data of foreign
+ * transaction follows a locking model based on four linked concepts:
+ *
+ * * All FdwXact fields except for indoubt, inprocessing and status are protected
+ *   by FdwXactLock.  These three fields are protected by its mutex.
+ * * Setting held_by of an FdwXact entry means to own the FdwXact entry, which
+ *   prevent it from updated and removed by concurrent processes.
+ * * The FdwXact whose inprocessing is true is also not processed or removed
+ *   by concurrent processes.
+ * * A process who is going to process foreign transaction needs to hold its
+ *   FdwXact entry in advance.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk.  FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon.  We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts.  If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to disk.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED)
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define ServerSupportTransactionCallack(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->commit_foreign_xact_fn != NULL)
+#define SeverSupportTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/*
+ * Structure to bundle the foreign transaction participant.  This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry is
+	 * not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer *server;
+	UserMapping *usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char	   *fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool		modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function prepare_foreign_xact_fn;
+	CommitForeignTransaction_function commit_foreign_xact_fn;
+	RollbackForeignTransaction_function rollback_foreign_xact_fn;
+	GetPrepareId_function get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transactions involved in the transaction.  A member of
+ * participants may not support transaction callbacks: commit, rollback and
+ * prepare.  If a member of participants doesn't support any transaction
+ * callbacks, i.g. ServerSupportTransactionCallack() returns false,
+ * we don't end its transaction.
+ *
+ * FdwXactParticipants_tmp is used to update FdwXactParticipants atomically
+ * when executing COMMIT/ROLLBACK PREPARED command.  In COMMIT PREPARED case,
+ * we don't want to rollback foreign transactions even if an error occurs,
+ * because the local prepared transaction never turn over rollback in that
+ * case.  However, preparing FdwXactParticipants might be lead an error
+ * because of calling palloc() inside.  So we prepare FdwXactParticipants in
+ * two phase.  In the first phase, PrepareFdwXactParticipants(), we collect
+ * all foreign transactions associated with the local prepared transactions
+ * and kept them in FdwXactParticipants_tmp.  Even if an error occurs during
+ * that, we don't rollback them.  In the second phase, SetFdwXactParticipants(),
+ * we replace FdwXactParticipants_tmp with FdwXactParticipants and hold them.
+ *
+ * FdwXactLocalXid is the local transaction id associated with FdwXactParticipants.
+ */
+static List *FdwXactParticipants = NIL;
+static List *FdwXactParticipants_tmp = NIL;
+static TransactionId FdwXactLocalXid = InvalidTransactionId;
+
+/*
+ * True is the current transaction needs to be committed together with
+ * foreign servers.
+ */
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int			max_prepared_foreign_xacts = 0;
+int			max_foreign_xact_resolvers = 0;
+int			foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static void FdwXactParticipantEndTransaction(FdwXactParticipant *fdw_part, bool onephase,
+											 bool for_commit);
+static bool checkForeignTwophaseCommitRequired(void);
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static void FdwXactPrepareForeignTransactions(void);
+static void FdwXactResolveOneFdwXact(FdwXact fdwxact);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(void);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid, void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static TransactionId FdwXactDetermineTransactionFate(TransactionId xid);
+static int	get_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						bool hold);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXactParticipant *create_fdwxact_participant(Oid serverid, Oid userid,
+													  FdwRoutine *routine);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&fdwxacts[cnt].mutex);
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation	rel;
+	Oid			serverid;
+	Oid			userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction. The foreign transaction identified
+ * by given server id and user id.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant *fdw_part;
+	MemoryContext old_ctx;
+	FdwRoutine *routine;
+	ListCell   *lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	fdw_part = create_fdwxact_participant(serverid, userid, routine);
+	fdw_part->modified = modified;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	pfree(routine);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/* Return palloc'd FdwXactParticipant variable */
+static FdwXactParticipant *
+create_fdwxact_participant(Oid serverid, Oid userid, FdwRoutine *routine)
+{
+	FdwXactParticipant *fdw_part;
+	ForeignServer *foreign_server;
+	UserMapping *user_mapping;
+
+	foreign_server = GetForeignServer(serverid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->modified = false;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	return fdw_part;
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * When foreign twophase commit is enabled, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign servers' FDW to support two-phase commit protocol and ask them to
+ * prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and when 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXacts(void)
+{
+	ListCell   *lc;
+	bool		need_twophase_commit;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false if
+	 * foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = checkForeignTwophaseCommitRequired();
+
+	/*
+	 * Prepare foreign transactions on foreign servers that support two-phase
+	 * commit.
+	 */
+	if (need_twophase_commit)
+	{
+		FdwXactPrepareForeignTransactions();
+		ForeignTwophaseCommitIsRequired = true;
+	}
+
+	/*
+	 * Commit other foreign transactions and delete the participant entry from
+	 * the list.
+	 */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		/*
+		 * Skip already prepared foreign transactions. Note that we keep those
+		 * FdwXactParticipants until the end of the transaction.
+		 */
+		if (fdw_part->fdwxact)
+			continue;
+
+		/* Delete non-transaction-support participants */
+		if (!ServerSupportTransactionCallack(fdw_part))
+		{
+			FdwXactParticipants = foreach_delete_current(FdwXactParticipants, lc);
+			continue;
+		}
+
+		/* Commit the foreign transaction in one-phase */
+		FdwXactParticipantEndTransaction(fdw_part, true, true);
+
+		/* Transaction successfully committed delete from the participant list */
+		FdwXactParticipants = foreach_delete_current(FdwXactParticipants, lc);
+	}
+}
+
+/*
+ * Return true if the current transaction modifies data on two or more servers
+ * in FdwXactParticipants and local server itself.
+ */
+static bool
+checkForeignTwophaseCommitRequired(void)
+{
+	ListCell   *lc;
+	bool		need_twophase_commit;
+	bool		have_notwophase;
+	int			nserverswritten = 0;
+	int			nserverstwophase = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (!fdw_part->modified)
+			continue;
+
+		if (SeverSupportTwophaseCommit(fdw_part))
+			nserverstwophase++;
+
+		nserverswritten++;
+	}
+
+	/* check if there is a server that doesn't support two-phase commit */
+	have_notwophase = (nserverswritten != nserverstwophase);
+
+	/* Did we modify the local non-temporary data? */
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		nserverswritten++;
+
+	if (nserverswritten <= 1)
+		return false;
+
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED)
+	{
+		/*
+		 * In 'required' case, we require for all modified server to support
+		 * two-phase commit.
+		 */
+		need_twophase_commit = (nserverswritten >= 2);
+	}
+	else
+	{
+		Assert(foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER);
+
+		/*
+		 * In 'prefer' case, we prepare transactions on only servers that
+		 * capable of two-phase commit.
+		 */
+		need_twophase_commit = (nserverstwophase >= 2);
+	}
+
+	/*
+	 * If foreign two phase commit is required then all foreign serves must be
+	 * capable of doing two-phase commit
+	 */
+	if (need_twophase_commit)
+	{
+		/* Parameter check */
+		if (max_prepared_foreign_xacts == 0)
+			ereport(ERROR,
+					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					 errmsg("foreign two-phase commit is required but prepared foreign transactions are disabled"),
+					 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+		if (max_foreign_xact_resolvers == 0)
+			ereport(ERROR,
+					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					 errmsg("foreign two-phase commit is required but prepared foreign transactions are disabled"),
+					 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+		if (have_notwophase &&
+			foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					 errmsg("cannot process a distributed transaction that has operated on a foreign server"),
+					 errdetail("foreign_twophase_commit is \'required\' but the transaction has some foreign servers which are not capable of two-phase commit")));
+	}
+
+	return need_twophase_commit;
+}
+
+/*
+ * The routine for committing or rolling back the given transaction participant.
+ */
+static void
+FdwXactParticipantEndTransaction(FdwXactParticipant *fdw_part, bool onephase,
+								 bool for_commit)
+{
+	FdwXactRslvState state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state.xid = FdwXactLocalXid;
+	state.server = fdw_part->server;
+	state.usermapping = fdw_part->usermapping;
+	state.fdwxact_id = onephase ? NULL : fdw_part->fdwxact_id;
+	state.flags = onephase ? FDWXACT_FLAG_ONEPHASE : 0;
+
+	if (for_commit)
+	{
+		fdw_part->commit_foreign_xact_fn(&state);
+		elog(DEBUG1, "successfully committed the foreign transaction for server %u user %u",
+			 fdw_part->usermapping->serverid,
+			 fdw_part->usermapping->userid);
+	}
+	else
+	{
+		fdw_part->rollback_foreign_xact_fn(&state);
+		elog(DEBUG1, "successfully rolled back the foreign transaction for server %u user %u",
+			 fdw_part->usermapping->serverid,
+			 fdw_part->usermapping->userid);
+	}
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here on failure. If any error occurs, we
+ * rollback non-prepared foreign transactions.
+ */
+static void
+FdwXactPrepareForeignTransactions(void)
+{
+	ListCell   *lc;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Save the local transaction id */
+	FdwXactLocalXid = GetTopTransactionId();
+
+	/* Loop over the foreign connections */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		FdwXactRslvState state;
+		FdwXact		fdwxact;
+
+		if (!SeverSupportTwophaseCommit(fdw_part) || !fdw_part->modified)
+			continue;
+
+		/* Get prepared transaction identifier */
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, FdwXactLocalXid);
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the
+		 * FDWXACT_STATUS_PREPARING status. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before
+		 * persisting the information to the disk and crash in-between these
+		 * two steps, we will lost the prepared transaction on the foreign
+		 * server and will not be able to resolve it after the crash recovery.
+		 * Hence persist first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(FdwXactLocalXid, fdw_part);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 */
+		state.xid = FdwXactLocalXid;
+		state.server = fdw_part->server;
+		state.usermapping = fdw_part->usermapping;
+		state.fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+		fdw_part->prepare_foreign_xact_fn(&state);
+
+		/* succeeded, update status */
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		SpinLockRelease(&fdwxact->mutex);
+	}
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact		fdwxact;
+	FdwXactOnDiskData *fdwxact_file_data;
+	MemoryContext old_context;
+	int			data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							 fdw_part->usermapping->userid,
+							 fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_PREPARING;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyProc->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyProc->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			   Oid umid, char *fdwxact_id)
+{
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->valid &&
+			fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("Duplicate entry with transaction id %u, serverid %u, userid %u exists.",
+									  xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->local_xid = xid;
+	fdwxact->dbid = dbid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->inprocessing = false;
+	fdwxact->indoubt = false;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int			i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("Failed to find entry for xid %u, foreign server %u, and user %u.",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the WAL
+		 * record is inserted could complete without fsync'ing our state file.
+		 * (This is essentially the same kind of race condition as the
+		 * COMMIT-to-clog-write case that RecordTransactionCommit uses
+		 * delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyProc->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and remove
+		 * the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/* Now we can mark ourselves as out of the commit critical section */
+		MyProc->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	   *id;
+	int			id_len = 0;
+
+	/*
+	 * If FDW doesn't provide the callback function, generate an unique
+	 * identifier.
+	 */
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char		buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+				 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifier \"%s\" is too long",
+						id),
+				 errdetail("Foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * Prepare foreign transactions by PREPARE TRANSACTION command.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXacts(void)
+{
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	if (!checkForeignTwophaseCommitRequired())
+		return;
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions();
+
+	/*
+	 * Prepared foreign transactions need to be resolved when COMMIT PREPARED
+	 * or ROLLBACK PREPARED.  Therefore we forget all participants here so
+	 * that we don't mark them as in-doubt at the end of the transaction on
+	 * failure.
+	 */
+	list_free(FdwXactParticipants);
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Collect all foreign transactions associated with the given xid.  Return true
+ * if COMMIT PREPARED or ROLLBACK PREPARED needs to wait for all foreign transactions
+ * to be resolved.  The collected foreign transactions are kept in FdwXactParticipants_tmp,
+ * so the caller must call SetFdwXactParticipants() later if this function returns true.
+ */
+bool
+PrepareFdwXactParticipants(TransactionId xid)
+{
+	MemoryContext old_ctx;
+
+	Assert(FdwXactParticipants_tmp == NIL);
+
+	if (!TwoPhaseExists(xid))
+		return false;
+
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXactParticipant *fdw_part;
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		FdwRoutine *routine;
+
+		if (!fdwxact->valid || fdwxact->local_xid != xid)
+			continue;
+
+		routine = GetFdwRoutineByServerId(fdwxact->serverid);
+		fdw_part = create_fdwxact_participant(fdwxact->serverid, fdwxact->userid,
+											  routine);
+		fdw_part->modified = true;
+		fdw_part->fdwxact = fdwxact;
+
+		/* Add to the participants list */
+		FdwXactParticipants_tmp = lappend(FdwXactParticipants_tmp, fdw_part);
+	}
+	LWLockRelease(FdwXactLock);
+
+	MemoryContextSwitchTo(old_ctx);
+
+	/*
+	 * We cannot proceed to commit this prepared transaction when
+	 * foreign_twophase_commit is disabled.
+	 */
+	if (FdwXactParticipants_tmp != NIL &&
+		!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot process a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	return (FdwXactParticipants_tmp != NIL);
+}
+
+/*
+ * Make the collected foreign transactions the participants of this transaction and
+ * hold all of them.  This function must be called after PrepareFdwXactParticipants().
+ */
+void
+SetFdwXactParticipants(TransactionId xid, bool commit)
+{
+	ListCell   *lc;
+
+	Assert(FdwXactParticipants_tmp != NIL);
+	Assert(FdwXactParticipants == NIL);
+
+	FdwXactLocalXid = xid;
+	FdwXactParticipants = FdwXactParticipants_tmp;
+	FdwXactParticipants_tmp = NIL;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		Assert(SeverSupportTwophaseCommit(fdw_part));
+
+		/* Hold the fdwxact entry and set the status */
+		SpinLockAcquire(&fdw_part->fdwxact->mutex);
+		Assert(fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED);
+		fdw_part->fdwxact->held_by = MyBackendId;
+		fdw_part->fdwxact->status = commit
+			? FDWXACT_STATUS_COMMITTING
+			: FDWXACT_STATUS_ABORTING;
+		SpinLockRelease(&fdw_part->fdwxact->mutex);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants();
+}
+
+/*
+ * Wait for its all foreign transactions to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitForResolution(TransactionId wait_xid)
+{
+	char	   *new_status = NULL;
+	const char *old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/*
+	 * Quick exit if either atomic commit is not requested or we don't have
+	 * any participants.
+	 */
+	if (!IsForeignTwophaseCommitRequested() || FdwXactParticipants == NIL)
+		return;
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction resolution.
+	 */
+	if (update_process_title)
+	{
+		int			len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status);
+		new_status[len] = '\0'; /* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The
+		 * latter would lead the client to believe that the distributed
+		 * transaction aborted, which is not true: it's already committed
+		 * locally. The former is no good either: the client has requested
+		 * committing a distributed transaction, and is entitled to assume
+		 * that a acknowledged commit is also commit on all foreign servers,
+		 * which might not be true. So in this case we issue a WARNING (which
+		 * some clients may be able to interpret) and shut off further output.
+		 * We do NOT reset PorcDiePending, so that the process will die after
+		 * the commit is cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve
+		 * them later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status);
+		pfree(new_status);
+	}
+
+	list_free(FdwXactParticipants);
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz now, TimestampTz *nextResolutionTs_p,
+				 TransactionId *waitXid_p)
+{
+	PGPROC	   *proc;
+	bool		found = false;
+
+	Assert(LWLockHeldByMe(FdwXactResolutionLock));
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	/* Initialize variables */
+	*nextResolutionTs_p = -1;
+	*waitXid_p = InvalidTransactionId;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+		{
+			if (proc->fdwXactNextResolutionTs <= now)
+			{
+				/* Found a waiting process */
+				found = true;
+				*waitXid_p = proc->fdwXactWaitXid;
+			}
+			else
+				/* Found a waiting process supposed to be processed later */
+				*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+
+			break;
+		}
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return found ? proc : NULL;
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC	   *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC	   *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC	   *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * In abort case, this function ends foreign transaction participants and possibly
+ * rollback their prepared foreign trasnactions.
+ */
+extern void
+AtEOXact_FdwXacts(bool is_commit)
+{
+	ListCell   *lc;
+
+	if (!is_commit)
+	{
+		foreach(lc, FdwXactParticipants)
+		{
+			FdwXactParticipant *fdw_part = lfirst(lc);
+			FdwXact		fdwxact = fdw_part->fdwxact;
+			int			status;
+
+			if (!fdwxact)
+			{
+				/* Rollback foreign transaction in one-phase if supported */
+				if (ServerSupportTransactionCallack(fdw_part))
+					FdwXactParticipantEndTransaction(fdw_part, true, false);
+				continue;
+			}
+
+			/*
+			 * Abort the foreign transaction.  For participants whose status
+			 * is FDWXACT_STATUS_PREPARING, we close the transaction in
+			 * one-phase. In addition, since we are not sure that the
+			 * preparation has been completed on the foreign server, we also
+			 * attempts to rollback the prepared foreign transaction.  Note
+			 * that it's FDWs responsibility that they tolerate
+			 * OBJECT_NOT_FOUND error in abort case.
+			 */
+			SpinLockAcquire(&fdwxact->mutex);
+			status = fdwxact->status;
+			fdwxact->status = FDWXACT_STATUS_ABORTING;
+			SpinLockRelease(&fdwxact->mutex);
+
+			switch (status)
+			{
+				case FDWXACT_STATUS_PREPARING:
+					/* One-phase rollback foreign transaction */
+					FdwXactParticipantEndTransaction(fdw_part, true, false);
+					/* fall through */
+				case FDWXACT_STATUS_PREPARED:
+				case FDWXACT_STATUS_ABORTING:
+					/* One-phase rollback foreign transaction */
+					FdwXactParticipantEndTransaction(fdw_part, false, false);
+					break;
+				case FDWXACT_STATUS_COMMITTING:
+					Assert(false);
+					break;
+			}
+
+			/* Resolution was a success, remove the entry */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			if (fdwxact->ondisk)
+				RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								  fdwxact->serverid, fdwxact->userid,
+								  true);
+			remove_fdwxact(fdwxact);
+			LWLockRelease(FdwXactLock);
+		}
+
+		/* All foreign transaction should be aborted */
+		list_free(FdwXactParticipants);
+		FdwXactParticipants = NIL;
+	}
+
+	ForgetAllFdwXactParticipants();
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(void)
+{
+	ListCell   *cell;
+	int			nlefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact		fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		Assert(TransactionIdIsValid(FdwXactLocalXid));
+		Assert(fdwxact);
+
+		/*
+		 * Unlock and mark a foreign transaction as in-doubt.  Note that there
+		 * is a race condition; the FdwXact entries in FdwXactParticipants
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget. So we need to check if the entries are still
+		 * associated with the transaction.  Also we do these check by
+		 * transaction id because these foreign transaction may already be
+		 * held by the resolver.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		if (fdwxact->valid && fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+			fdwxact->indoubt = true;	/* let resolver to process */
+			nlefts++;
+		}
+		LWLockRelease(FdwXactLock);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (nlefts > 0)
+	{
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", nlefts);
+		FdwXactComputeRequiredXmin();
+		FdwXactLaunchOrWakeupResolver();
+	}
+
+	list_free(FdwXactParticipants);
+	FdwXactParticipants = NIL;
+	FdwXactParticipants_tmp = NIL;
+	FdwXactLocalXid = InvalidTransactionId;
+}
+
+/*
+ * Resolve foreign transactions at the give indexes. If 'waiter' is not NULL,
+ * we release the waiter after we resolved all of the given foreign transactions
+ * On failure we re-enqueue the waiting backend after incremented the next
+ * resolution time.
+ *
+ * The caller must hold the given foreign transactions in advance to prevent
+ * concurrent update.
+ */
+void
+FdwXactResolveFdwXacts(int *fdwxact_idxs, int nfdwxacts, PGPROC *waiter)
+{
+	for (int i = 0; i < nfdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[fdwxact_idxs[i]];
+
+		CHECK_FOR_INTERRUPTS();
+
+		PG_TRY();
+		{
+			FdwXactResolveOneFdwXact(fdwxact);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			if (waiter)
+			{
+				LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+				if (waiter->fdwXactState == FDWXACT_WAITING)
+				{
+					SHMQueueDelete(&(waiter->fdwXactLinks));
+					pg_write_barrier();
+					waiter->fdwXactNextResolutionTs =
+						TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+													foreign_xact_resolution_retry_interval);
+					FdwXactQueueInsert(waiter);
+				}
+				LWLockRelease(FdwXactResolutionLock);
+			}
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		if (fdwxact->ondisk)
+			RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+							  fdwxact->userid, true);
+		remove_fdwxact(fdwxact);
+		LWLockRelease(FdwXactLock);
+	}
+
+	if (!waiter)
+		return;
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter could
+	 * already be detached if user cancelled to wait before resolution.
+	 */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/*
+		 * Wake up the waiter only when we have set state and removed from
+		 * queue
+		 */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+FdwXactExists(Oid dbid, Oid serverid, Oid userid)
+{
+	int			idx;
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	idx = get_fdwxact(dbid, InvalidTransactionId, serverid, userid,
+					  false);
+	LWLockRelease(FdwXactLock);
+
+	return (idx != -1);
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ *
+ * XXX: we can exclude FdwXact entries whose status is already committing
+ * or aborting.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+
+/*
+ * Determine whether the foreign transaction associated with the given transaction
+ * id should be committed or rolled back according to the result of the local
+ * transaction.
+ */
+static FdwXactStatus
+FdwXactDetermineTransactionFate(TransactionId xid)
+{
+	/*
+	 * If the local transaction is already committed, commit prepared foreign
+	 * transaction.
+	 */
+	if (TransactionIdDidCommit(xid))
+		return FDWXACT_STATUS_COMMITTING;
+
+	/*
+	 * If the local transaction is already aborted, abort prepared foreign
+	 * transactions.
+	 */
+	else if (TransactionIdDidAbort(xid))
+		return FDWXACT_STATUS_ABORTING;
+
+
+	/*
+	 * The local transaction is not in progress but the foreign transaction is
+	 * not prepared on the foreign server. This can happen when transaction
+	 * failed after registered this entry but before actual preparing on the
+	 * foreign server. So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(xid))
+		return FDWXACT_STATUS_ABORTING;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is about
+	 * to be committed or aborted. This should not happen except for one case
+	 * where the local transaction is prepared and this foreign transaction is
+	 * being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		elog(ERROR,
+			 "cannot resolve the foreign transaction associated with in-process transaction");
+
+	pg_unreachable();
+}
+
+/*
+ * Commit or rollback one prepared foreign transaction.  After resolved
+ * successfully, the FdwXact entry is removed from the shared memory and also
+ * remove the corresponding on-disk file.
+ */
+static void
+FdwXactResolveOneFdwXact(FdwXact fdwxact)
+{
+	FdwXactRslvState state;
+	ForeignServer *server;
+	ForeignDataWrapper *fdw;
+	FdwRoutine *routine;
+
+	Assert(fdwxact != NULL);
+	Assert(fdwxact->held_by != InvalidBackendId || fdwxact->inprocessing);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+	{
+		FdwXactStatus new_status;
+
+		new_status = FdwXactDetermineTransactionFate(fdwxact->local_xid);
+		Assert(new_status == FDWXACT_STATUS_COMMITTING ||
+			   new_status == FDWXACT_STATUS_ABORTING);
+
+		/* Update the status */
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwxact->status = new_status;
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	routine = GetFdwRoutine(fdw->fdwhandler);
+
+	/* Prepare resolution state to pass to API */
+	state.xid = fdwxact->local_xid;
+	state.server = server;
+	state.usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+	state.fdwxact_id = fdwxact->fdwxact_id;
+	state.flags = 0;
+
+	if (fdwxact->status == FDWXACT_STATUS_COMMITTING)
+	{
+		routine->CommitForeignTransaction(&state);
+		elog(DEBUG1, "successfully committed the prepared foreign transaction for server %u user %u",
+			 fdwxact->serverid, fdwxact->userid);
+	}
+	else
+	{
+		routine->RollbackForeignTransaction(&state);
+		elog(DEBUG1, "successfully rolled back the prepared foreign transaction for server %u user %u",
+			 fdwxact->serverid, fdwxact->userid);
+	}
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respectively.
+ */
+static int
+get_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid, bool hold)
+{
+	bool		found = false;
+	int			i;
+
+	Assert(LWLockHeldByMe(FdwXactLock));
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		bool		inprocessing;
+
+		if (!fdwxact->valid)
+			continue;
+
+		SpinLockAcquire(&fdwxact->mutex);
+		inprocessing = fdwxact->inprocessing;
+		SpinLockRelease(&fdwxact->mutex);
+
+		/*
+		 * If we're attempting to hold this entry, skip if it is already held
+		 * or being processed.
+		 */
+		if (hold &&
+			(inprocessing || fdwxact->held_by != InvalidBackendId))
+			continue;
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* This entry matches the condition */
+
+		if (hold)
+			fdwxact->held_by = MyBackendId;
+
+		found = true;
+		break;
+	}
+
+	return found ? i : -1;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record in
+		 * FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transaction log record", info);
+
+	return;
+}
+
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The status of
+	 * the transaction is set as preparing, since we do not know the exact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction which prepared this foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							 fdwxact_data->serverid, fdwxact_data->userid,
+							 fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know the xact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;		/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact		fdwxact;
+	int			i;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->dbid == dbid && fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid && fdwxact->userid == userid)
+			break;
+	}
+
+	if (i >= FdwXactCtl->num_fdwxacts)
+		return;
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint's redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;					/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence of long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked invalid,
+	 * because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+				(errmsg_plural("%u foreign transaction state file was written "
+							   "for long-running prepared transactions",
+							   "%u foreign transaction state files were written "
+							   "for long-running prepared transactions",
+							   serialized_fdwxacts,
+							   serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
+									XL_ROUTINE(.page_read = &read_local_xlog_page,
+											   .segment_open = &wal_segment_open,
+											   .segment_close = &wal_segment_close),
+									NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+				 errdetail("Failed while allocating an XLog reading processor.")));
+
+	XLogBeginRead(xlogreader, lsn);
+	record = XLogReadRecord(xlogreader, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not read foreign transaction state from xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not recreate foreign transaction state file \"%s\": %m",
+						path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId origNextXid =
+	XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	   *buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open FDW transaction state file \"%s\": %m",
+						path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char	   *buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign prepared transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+}			WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	6
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		FdwXactStatus status;
+		bool		indoubt;
+		char	   *xact_status;
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+
+		if (!fdwxact->valid)
+			continue;
+
+		memset(nulls, 0, sizeof(nulls));
+
+		SpinLockAcquire(&fdwxact->mutex);
+		status = fdwxact->status;
+		indoubt = fdwxact->indoubt;
+		SpinLockRelease(&fdwxact->mutex);
+
+		values[0] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[1] = ObjectIdGetDatum(fdwxact->serverid);
+		values[2] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (status)
+		{
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[3] = CStringGetTextDatum(xact_status);
+		values[4] = BoolGetDatum(indoubt);
+		values[5] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
+
+/*
+ * Built-in SQL function to resolve a prepared foreign transaction.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			serverid = PG_GETARG_OID(1);
+	Oid			userid = PG_GETARG_OID(2);
+	int			idx;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/* Find and hold the FdwXact entry */
+	idx = get_fdwxact(MyDatabaseId, xid, serverid, userid, true);
+
+	LWLockRelease(FdwXactLock);
+
+	if (idx < 0)
+	{
+		/* No entry */
+		PG_RETURN_BOOL(false);
+	}
+
+	PG_TRY();
+	{
+		FdwXactResolveFdwXacts(&idx, 1, NULL);
+	}
+	PG_CATCH();
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[idx];
+
+		SpinLockAcquire(&fdwxact->mutex);
+		FdwXactCtl->fdwxacts[idx]->held_by = InvalidBackendId;
+		SpinLockRelease(&fdwxact->mutex);
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			serverid = PG_GETARG_OID(1);
+	Oid			userid = PG_GETARG_OID(2);
+	FdwXact		fdwxact;
+	int			i;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->valid && fdwxact->dbid == MyDatabaseId &&
+			fdwxact->local_xid == xid && fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			break;
+	}
+
+	if (i >= FdwXactCtl->num_fdwxacts)
+	{
+		LWLockRelease(FdwXactLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("does not exist foreign transaction on server %u",
+						fdwxact->serverid)));
+	}
+
+	if (fdwxact->inprocessing || fdwxact->held_by != InvalidBackendId)
+	{
+		LWLockRelease(FdwXactLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot remove foreign transaction entry which is being processed")));
+	}
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000000..fed2fbcd08
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,560 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool		found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int			slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+		FdwXactRslvCtl->launcher_pid = InvalidPid;
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			memset(resolver, 0, sizeof(FdwXactResolver));
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == InvalidPid);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz now;
+		long		wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int			rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a
+		 * foreign_xact_resolution_retry_interval but always starts when the
+		 * backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool		launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested but not
+			 * running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we should
+			 * retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool		found = false;
+	int			i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the same
+	 * database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find
+		 * FdwXact entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int			unused_slot = -1;
+	int			i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign transaction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to
+	 * wait until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	   *resolver_dbs;	/* DBs resolver's running on */
+	HTAB	   *fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL		ctl;
+	HASH_SEQ_STATUS status;
+	Oid		   *entry;
+	bool		launched;
+	int			i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		bool		indoubt;
+		BackendId	held_by;
+
+		if (!fdwxact->valid)
+			continue;
+
+		SpinLockAcquire(&fdwxact->mutex);
+		indoubt = fdwxact->indoubt;
+		held_by = fdwxact->held_by;
+		SpinLockRelease(&fdwxact->mutex);
+
+		if ((indoubt && held_by == InvalidBackendId) ||
+			(!indoubt && held_by != InvalidBackendId))
+			hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+	{
+		hash_destroy(fdwxact_dbs);
+		return false;
+	}
+
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool		found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	hash_destroy(fdwxact_dbs);
+	hash_destroy(resolver_dbs);
+
+	return launched;
+}
+
+/* Register a background worker running the foreign transaction launcher */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid			dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int			i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign transaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int			rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		/* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+					   10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000000..b91a2e1e88
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,436 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int			foreign_xact_resolution_retry_interval;
+int			foreign_xact_resolver_timeout = 60 * 1000;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+static void hold_fdwxacts(PGPROC *waiter);
+static void hold_indoubt_fdwxacts(void);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+static TimestampTz last_resolution_time = -1;
+
+/*
+ * held_fdwxacts has indexes of FdwXact which the resolver marked
+ * as in-processing. We clear that flag from those entries on failure.
+ */
+static int *held_fdwxacts = NULL;
+static int	nheld;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	/* clear inprocessing flags */
+	for (int i = 0; i < nheld; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[held_fdwxacts[i]];
+
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwxact->inprocessing = false;
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int			slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+	CommitTransactionCommand();
+
+	held_fdwxacts = palloc(sizeof(int) * max_prepared_foreign_xacts);
+	nheld = 0;
+
+	/* Initialize stats to a sanish value */
+	last_resolution_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TransactionId waitXid = InvalidTransactionId;
+		TimestampTz resolutionTs = -1;
+		TimestampTz now;
+		int			rc;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or the queue has
+		 * only waiters that have a future resolution timestamp.
+		 */
+		for (;;)
+		{
+			PGPROC	   *waiter;
+
+			CHECK_FOR_INTERRUPTS();
+
+			LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+
+			waiter = FdwXactGetWaiter(now, &resolutionTs, &waitXid);
+
+			if (!waiter)
+			{
+				/* Not found, break */
+				LWLockRelease(FdwXactResolutionLock);
+				break;
+			}
+
+			/* Hold the waiting foreign transactions */
+			hold_fdwxacts(waiter);
+			Assert(nheld > 0);
+			LWLockRelease(FdwXactResolutionLock);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveFdwXacts(held_fdwxacts, nheld, waiter);
+			CommitTransactionCommand();
+
+			last_resolution_time = now;
+		}
+
+		/* Hold in-doubt transactions */
+		hold_indoubt_fdwxacts();
+
+		if (nheld > 0)
+		{
+			StartTransactionCommand();
+			FdwXactResolveFdwXacts(held_fdwxacts, nheld, NULL);
+			CommitTransactionCommand();
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(last_resolution_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		/* There is no waiting backend */
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * FdwXactWaiterExists check.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long		sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(last_resolution_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
+
+/*
+ * Mark in-doubt transactions as in-processing.
+ */
+static void
+hold_indoubt_fdwxacts(void)
+{
+	nheld = 0;
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->valid && fdwxact->dbid == MyDatabaseId &&
+			fdwxact->held_by == InvalidBackendId && fdwxact->indoubt)
+		{
+			held_fdwxacts[nheld++] = i;
+
+			/* hold lock */
+			SpinLockAcquire(&fdwxact->mutex);
+			fdwxact->inprocessing = true;
+			SpinLockRelease(&fdwxact->mutex);
+		}
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Mark foreign transactions associated with the given waiter's transaction
+ * as in-processing.
+ */
+static void
+hold_fdwxacts(PGPROC *waiter)
+{
+	nheld = 0;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->valid && fdwxact->dbid == waiter->databaseId &&
+			fdwxact->local_xid == waiter->fdwXactWaitXid)
+		{
+			held_fdwxacts[nheld++] = i;
+
+			/* hold lock */
+			SpinLockAcquire(&fdwxact->mutex);
+			Assert(!fdwxact->indoubt);
+			Assert(fdwxact->held_by = waiter->backendId);
+			fdwxact->inprocessing = true;
+			SpinLockRelease(&fdwxact->mutex);
+		}
+	}
+	LWLockRelease(FdwXactLock);
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index f88d72fd86..982c1a36cc 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -13,6 +13,7 @@ OBJS = \
 	clogdesc.o \
 	committsdesc.o \
 	dbasedesc.o \
+	fdwxactdesc.o \
 	genericdesc.o \
 	gindesc.o \
 	gistdesc.o \
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000000..ca761763e5
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxactdesc.c
+ *		PostgreSQL global transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdwxactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdwxact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		FdwXactOnDiskData *fdwxact_insert = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_insert->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_insert->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_insert->dbid);
+		appendStringInfo(buf, " local xid: %u,", fdwxact_insert->local_xid);
+		appendStringInfo(buf, " id: %s", fdwxact_insert->fdwxact_id);
+	}
+	else
+	{
+		xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_remove->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_remove->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_remove->dbid);
+		appendStringInfo(buf, " local xid: %u", fdwxact_remove->xid);
+	}
+
+}
+
+const char *
+fdwxact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDWXACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDWXACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 1cd97852e8..ea045174e0 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 58091f6b52..200cf9d067 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -10,6 +10,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index e1904877fa..2b9e039580 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -850,6 +851,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 	return result;
 }
 
+/*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
 /*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
@@ -2196,6 +2226,9 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	XLogRecPtr	recptr;
 	TimestampTz committs = GetCurrentTimestamp();
 	bool		replorigin;
+	bool		need_fdwxact_commit;
+
+	need_fdwxact_commit = PrepareFdwXactParticipants(xid);
 
 	/*
 	 * Are we using the replication origins feature?  Or, in other words, are
@@ -2266,6 +2299,16 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	if (need_fdwxact_commit)
+	{
+		SetFdwXactParticipants(xid, true);
+		FdwXactWaitForResolution(xid);
+	}
 }
 
 /*
@@ -2285,6 +2328,9 @@ RecordTransactionAbortPrepared(TransactionId xid,
 							   const char *gid)
 {
 	XLogRecPtr	recptr;
+	bool		need_fdwxact_commit;
+
+	need_fdwxact_commit = PrepareFdwXactParticipants(xid);
 
 	/*
 	 * Catch the scenario where we aborted partway through
@@ -2325,6 +2371,16 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be rolled back.
+	 */
+	if (need_fdwxact_commit)
+	{
+		SetFdwXactParticipants(xid, false);
+		FdwXactWaitForResolution(xid);
+	}
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 3984dd3e1a..d89fe5182e 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1219,6 +1220,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1227,6 +1229,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1265,12 +1268,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1428,6 +1432,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitForResolution(xid);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2087,6 +2099,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXacts();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2254,6 +2269,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXacts(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2341,6 +2357,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXacts();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2542,6 +2560,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	AtEOXact_FdwXacts(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2751,6 +2770,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXacts(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index a53e6d9633..c62d6aa3b7 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
@@ -4599,6 +4600,7 @@ InitControlFile(uint64 sysidentifier)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6289,6 +6291,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_wal_senders",
 									 max_wal_senders,
 									 ControlFile->max_wal_senders);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
@@ -6835,14 +6840,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -7044,7 +7050,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7557,6 +7566,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7887,6 +7897,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9182,6 +9195,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9711,8 +9725,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9730,6 +9746,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9746,6 +9763,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9951,6 +9969,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -10150,6 +10169,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 2bd5f5ea14..0dd403b588 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -333,6 +333,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index ac07f75bc3..e82630eefa 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2812,8 +2812,14 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
+	}
 
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index f197869752..6206265424 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1101,6 +1103,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (FdwXactExists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1419,6 +1433,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
+	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (FdwXactExists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
 	/*
 	 * Do the deletion
 	 */
@@ -1572,6 +1595,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index fb6ce49056..3fa8bfe09f 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/table.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
@@ -939,7 +940,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 513471ab9b..29f376e48c 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,9 +226,31 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
+	}
 
 	return scanstate;
 }
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 1ec07bad07..e5dee94764 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -2418,6 +2420,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index 61e48ca3f8..8f411c0559 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMappingOid - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,18 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction && !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction && routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper must support both commit and rollback routines or neither");
+
+	if (routine->PrepareForeignTransaction &&
+		!routine->CommitForeignTransaction &&
+		!routine->RollbackForeignTransaction)
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index beb5e85434..2258424e81 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -12,6 +12,8 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -128,6 +130,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 3f8105c6eb..2abd61b88a 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3661,6 +3661,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3873,6 +3879,11 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_SYNC_REP:
 			event_name = "SyncRep";
 			break;
+		case WAIT_EVENT_FDWXACT:
+			event_name = "FdwXact";
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
@@ -4097,6 +4108,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index e19d5dc1a6..b804b9ea41 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -909,6 +911,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -973,12 +979,13 @@ PostmasterMain(int argc, char *argv[])
 	LocalProcessControlFile(false);
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index c2e5e3abf8..9d34817f39 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -151,6 +151,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 427b0d59cd..55609eed81 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(void)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(void)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 363000670b..6a05070590 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -94,6 +94,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -249,6 +251,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1313,6 +1316,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1378,6 +1382,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1427,6 +1432,15 @@ GetOldestXmin(Relation rel, int flags)
 		NormalTransactionIdPrecedes(replication_slot_xmin, result))
 		result = replication_slot_xmin;
 
+	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
 	/*
 	 * After locks have been released and vacuum_defer_cleanup_age has been
 	 * applied, check whether we need to back up further to make logical
@@ -3127,6 +3141,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations of the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index db47843229..adb276370c 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -49,3 +49,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 CLogTruncationLock					44
+FdwXactLock							45
+FdwXactResolverLock					46
+FdwXactResolutionLock				47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index 5aa19d3f78..889dfa7e9a 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -421,6 +422,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* Initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -822,6 +827,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 8958ec8103..5ed6c05b18 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3054,6 +3056,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 5bdc02fce2..2558c50cef 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -426,6 +427,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 	{NULL, 0, false}
 };
 
+/*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
 /*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
@@ -763,6 +783,12 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FDWXACT */
+	gettext_noop("Foreign Transaction Management"),
+	/* FDWXACT_SETTINGS */
+	gettext_noop("Foreign Transaction Management / Settings"),
+	/* FDWXACT_RESOLVER */
+	gettext_noop("Foreign Transaction Management / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2471,6 +2497,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FDWXACT_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FDWXACT_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4599,6 +4671,16 @@ static struct config_enum ConfigureNamesEnum[] =
 		NULL, assign_synchronous_commit, NULL
 	},
 
+	{
+		{"foreign_twophase_commit", PGC_USERSET, FDWXACT_SETTINGS,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 995b6ca155..d7ca008a9e 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -344,6 +346,20 @@
 #max_sync_workers_per_subscription = 2	# taken from max_logical_replication_workers
 
 
+#------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+#foreign_twophase_commit = disabled	# use two-phase commit for distributed transactions:
+					# disabled, prefer or required
+
 #------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index a0b0458108..8701c5f005 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index a66dd078a7..00ca97b96b 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -208,6 +208,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index e73639df74..3041c39bc0 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -302,6 +302,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 233441837f..b040202043 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -710,6 +710,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -914,6 +915,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/fdwxactdesc.c b/src/bin/pg_waldump/fdwxactdesc.c
new file mode 120000
index 0000000000..ce8c21880c
--- /dev/null
+++ b/src/bin/pg_waldump/fdwxactdesc.c
@@ -0,0 +1 @@
+../../../src/backend/access/rmgrdesc/fdwxactdesc.c
\ No newline at end of file
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca4b1..b616cea347 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000000..bf745cb741
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,162 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL global transaction manager
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDWXACT_H
+#define FDWXACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* fdwXactState */
+#define	FDWXACT_NOT_WAITING		0
+#define	FDWXACT_WAITING			1
+#define	FDWXACT_WAIT_COMPLETE	2
+
+/* Flag passed to FDW transaction management APIs */
+#define FDWXACT_FLAG_ONEPHASE		0x01	/* transaction can commit/rollback
+											 * without preparation */
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER, /* use twophase commit where available */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support
+										 * twophase commit */
+}			ForeignTwophaseCommitLevel;
+
+/* Enum to track the status of foreign transaction */
+typedef enum
+{
+	FDWXACT_STATUS_INVALID = 0,
+	FDWXACT_STATUS_PREPARING,	/* foreign transaction is being prepared */
+	FDWXACT_STATUS_PREPARED,	/* foreign transaction is prepared */
+	FDWXACT_STATUS_COMMITTING,	/* foreign prepared transaction is being
+								 * committed */
+	FDWXACT_STATUS_ABORTING		/* foreign prepared transaction is being
+								 * aborted */
+} FdwXactStatus;
+
+typedef struct FdwXactData *FdwXact;
+
+/*
+ * Shared memory state of a single foreign transaction.
+ */
+typedef struct FdwXactData
+{
+	FdwXact		fdwxact_free_next;	/* Next free FdwXact entry */
+
+	TransactionId local_xid;	/* XID of local transaction */
+	Oid			dbid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			umid;
+
+	FdwXactStatus status;
+	bool		indoubt;		/* Is an in-doubt transaction? */
+	bool		inprocessing;	/* resolver is processing? */
+	slock_t		mutex;			/* protect above three fields */
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;	/* XLOG offset of inserting this entry
+									 * start */
+	XLogRecPtr	insert_end_lsn; /* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to
+								 * file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* prepared transaction
+												 * identifier */
+}			FdwXactData;
+
+/*
+ * Shared memory layout for maintaining foreign prepared transaction entries.
+ * Adding or removing FdwXact entry needs to hold FdwXactLock in exclusive mode,
+ * and iterating fdwXacts needs that in shared mode.
+ */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		free_fdwxacts;
+
+	/* Number of valid foreign transaction entries */
+	int			num_fdwxacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdwxacts[FLEXIBLE_ARRAY_MEMBER];	/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* State data for foreign transaction resolution, passed to FDW callbacks */
+typedef struct FdwXactRslvState
+{
+	TransactionId xid;
+
+	/* Foreign transaction information */
+	char	   *fdwxact_id;
+	ForeignServer *server;
+	UserMapping *usermapping;
+
+	int			flags;			/* OR of FDWXACT_FLAG_xx flags */
+} FdwXactRslvState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+/* Function declarations */
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void RegisterFdwXactByRelId(Oid relid, bool modified);
+extern void RegisterFdwXactByServerId(Oid serverid, bool modified);
+extern void PreCommit_FdwXacts(void);
+extern void FdwXactReleaseWaiter(PGPROC *waiter);
+extern void FdwXactWaitForResolution(TransactionId wait_xid);
+extern void FdwXactResolveFdwXacts(int *fdwxact_idxs, int nfdwxacts, PGPROC *waiter);
+extern PGPROC *FdwXactGetWaiter(TimestampTz now, TimestampTz *nextResolutionTs_p,
+								TransactionId *waitXid_p);
+extern bool FdwXactWaiterExists(Oid dbid);
+extern bool PrepareFdwXactParticipants(TransactionId xid);
+extern void SetFdwXactParticipants(TransactionId xid, bool commit);
+extern void AtEOXact_FdwXacts(bool is_commit);
+extern void AtPrepare_FdwXacts(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern bool FdwXactExists(Oid dboid, Oid serverid, Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+
+#endif							/* FDWXACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000000..688b43b8d0
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,28 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherRequestToLaunch(void);
+extern void FdwXactLaunchOrWakeupResolver(void);
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+
+
+#endif							/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000000..779848113c
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int	foreign_xact_resolver_timeout;
+
+#endif							/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000000..b4cec76eae
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,54 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDWXACT_INSERT	0x00
+#define XLOG_FDWXACT_REMOVE	0x10
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDWXACT_ID_MAX_LEN 200
+
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdwxact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+	bool		force;
+} xl_fdwxact_remove;
+
+extern void fdwxact_redo(XLogReaderState *record);
+extern void fdwxact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdwxact_identify(uint8 info);
+
+#endif							/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000000..c935471936
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,63 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/resolver_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t		pid;			/* this resolver's PID, or 0 if not active */
+	Oid			dbid;			/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool		in_use;
+
+	/* Protect shared variables shown above */
+	slock_t		mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	   *latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/* Foreign transaction resolution queue. Protected by FdwXactLock */
+	SHM_QUEUE	fdwxact_queue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch	   *launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+#define SizeOfFdwXactRslvCtlData \
+	(offsetof(FdwXactRslvCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif							/* RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 6c15df7e70..986bc73566 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Fdw Transaction", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 2ca71c3445..bd027a2861 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index a04fc70326..6f1f336e31 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -108,6 +108,13 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
+/*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index c8869d5226..da0d442f1b 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -232,6 +232,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index de5670e538..9884f5f8e7 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 4bce3ad8de..ce7e37b29e 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5983,6 +5983,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '9706', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o}',
+  proargnames => '{xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '9707', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '9708', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -6101,6 +6119,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '9709',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 95556dfb15..8d046cc4e4 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 5e0cf533fb..5596ee591c 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -69,6 +69,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 											   bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 														 bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index b8041d9988..d735959ff5 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -807,6 +807,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -887,7 +889,9 @@ typedef enum
 	WAIT_EVENT_REPLICATION_ORIGIN_DROP,
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
-	WAIT_EVENT_SYNC_REP
+	WAIT_EVENT_SYNC_REP,
+	WAIT_EVENT_FDWXACT,
+	WAIT_EVENT_FDWXACT_RESOLUTION
 } WaitEventIPC;
 
 /* ----------
@@ -970,6 +974,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index ae4f573ab4..f614908d44 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -16,6 +16,7 @@
 
 #include "access/clog.h"
 #include "access/xlogdefs.h"
+#include "datatype/timestamp.h"
 #include "lib/ilist.h"
 #include "storage/latch.h"
 #include "storage/lock.h"
@@ -154,6 +155,16 @@ struct PGPROC
 	int			syncRepState;	/* wait state for sync rep */
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
+	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
 	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index a5c7d0c064..0f73b64937 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -125,4 +127,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 454c2df487..6010dbcdee 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,9 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FDWXACT,
+	FDWXACT_SETTINGS,
+	FDWXACT_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index 8876025aaa..ddb4c1d3e8 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1342,6 +1342,13 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
-- 
2.23.0

diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
index fdc6b1f415..a0ac22d9eb 100644
--- a/src/backend/access/fdwxact/fdwxact.c
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -232,7 +232,7 @@ static bool fdwXactExitRegistered = false;
 static void register_fdwxact(Oid serverid, Oid userid, bool modified);
 static void FdwXactParticipantEndTransaction(FdwXactParticipant *fdw_part, bool onephase,
 											 bool for_commit);
-static bool checkForeignTwophaseCommitRequired(void);
+static bool checkForeignTwophaseCommitRequired(bool for_prepare);
 static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part);
 static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
 							  Oid userid, Oid umid, char *fdwxact_id);
@@ -464,7 +464,7 @@ PreCommit_FdwXacts(void)
 	 * Check if we need to use foreign twophase commit. It's always false if
 	 * foreign twophase commit is disabled.
 	 */
-	need_twophase_commit = checkForeignTwophaseCommitRequired();
+	need_twophase_commit = checkForeignTwophaseCommitRequired(false);
 
 	/*
 	 * Prepare foreign transactions on foreign servers that support two-phase
@@ -511,10 +511,10 @@ PreCommit_FdwXacts(void)
  * in FdwXactParticipants and local server itself.
  */
 static bool
-checkForeignTwophaseCommitRequired(void)
+checkForeignTwophaseCommitRequired(bool for_prepare)
 {
 	ListCell   *lc;
-	bool		need_twophase_commit;
+	bool		need_twophase_commit = false;
 	bool		have_notwophase;
 	int			nserverswritten = 0;
 	int			nserverstwophase = 0;
@@ -538,32 +538,43 @@ checkForeignTwophaseCommitRequired(void)
 	/* check if there is a server that doesn't support two-phase commit */
 	have_notwophase = (nserverswritten != nserverstwophase);
 
-	/* Did we modify the local non-temporary data? */
-	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
-		nserverswritten++;
-
-	if (nserverswritten <= 1)
-		return false;
-
-	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED)
+	if (for_prepare)
 	{
 		/*
-		 * In 'required' case, we require for all modified server to support
-		 * two-phase commit.
+		 * In case of PREPARE TRANSACTION we must use 2PC even when
+		 * only one foreign server is modified
 		 */
-		need_twophase_commit = (nserverswritten >= 2);
+		need_twophase_commit = (nserverswritten >= 1);
 	}
-	else
+
+	/* Did we modify the local non-temporary data? */
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		nserverswritten++;
+
+	if (!for_prepare)
 	{
-		Assert(foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER);
+		if (nserverswritten <= 1)
+			return false;
 
-		/*
-		 * In 'prefer' case, we prepare transactions on only servers that
-		 * capable of two-phase commit.
-		 */
-		need_twophase_commit = (nserverstwophase >= 2);
-	}
+		if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED)
+		{
+			/*
+			 * In 'required' case, we require for all modified server to support
+			 * two-phase commit.
+			 */
+			need_twophase_commit = (nserverswritten >= 2);
+		}
+		else
+		{
+			Assert(foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER);
 
+			/*
+			 * In 'prefer' case, we prepare transactions on only servers that
+			 * capable of two-phase commit.
+			 */
+			need_twophase_commit = (nserverstwophase >= 1);
+		}
+	}
 	/*
 	 * If foreign two phase commit is required then all foreign serves must be
 	 * capable of doing two-phase commit
@@ -589,6 +600,12 @@ checkForeignTwophaseCommitRequired(void)
 					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 					 errmsg("cannot process a distributed transaction that has operated on a foreign server"),
 					 errdetail("foreign_twophase_commit is \'required\' but the transaction has some foreign servers which are not capable of two-phase commit")));
+
+		if (have_notwophase && for_prepare)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					 errmsg("cannot process a distributed transaction that has operated on a foreign server"),
+					 errdetail("PREPARE TRANSACTION requires all foreign servers to support two-phase commit")));
 	}
 
 	return need_twophase_commit;
@@ -1001,7 +1018,7 @@ AtPrepare_FdwXacts(void)
 				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
 
-	if (!checkForeignTwophaseCommitRequired())
+	if (!checkForeignTwophaseCommitRequired(true))
 		return;
 
 	/* Prepare transactions on participating foreign servers. */
diff --git a/src/test/modules/test_fdwxact/expected/test_fdwxact.out b/src/test/modules/test_fdwxact/expected/test_fdwxact.out
index a5c8b89655..97f5052928 100644
--- a/src/test/modules/test_fdwxact/expected/test_fdwxact.out
+++ b/src/test/modules/test_fdwxact/expected/test_fdwxact.out
@@ -186,3 +186,41 @@ BEGIN;
 INSERT INTO t VALUES (1);
 INSERT INTO ft_no2pc_1 VALUES (1);
 COMMIT;
+SELECT count(*)  FROM pg_foreign_xacts();
+ count 
+-------
+     0
+(1 row)
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+INSERT INTO t VALUES(3);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+ count 
+-------
+     1
+(1 row)
+
+COMMIT PREPARED 'global_x1';
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+ count 
+-------
+     1
+(1 row)
+
+COMMIT PREPARED 'global_x1';
+--ERROR case
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+ERROR:  cannot process a distributed transaction that has operated on a foreign server
+DETAIL:  PREPARE TRANSACTION requires all foreign servers to support two-phase commit
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO t VALUES (1);
+COMMIT /*should do 2PC*/;
diff --git a/src/test/modules/test_fdwxact/sql/test_fdwxact.sql b/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
index 554312542f..271f651baf 100644
--- a/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
+++ b/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
@@ -160,10 +160,12 @@ BEGIN;
 INSERT INTO ft_1 VALUES (1);
 INSERT INTO ft_2pc_1 VALUES (1);
 COMMIT;
+
 BEGIN;
 INSERT INTO ft_no2pc_1 VALUES (1);
 INSERT INTO ft_2pc_1 VALUES (1);
 COMMIT;
+
 BEGIN;
 INSERT INTO ft_1 VALUES (1);
 INSERT INTO ft_2 VALUES (1);
@@ -176,3 +178,29 @@ BEGIN;
 INSERT INTO t VALUES (1);
 INSERT INTO ft_no2pc_1 VALUES (1);
 COMMIT;
+
+SELECT count(*)  FROM pg_foreign_xacts();
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+INSERT INTO t VALUES(3);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+COMMIT PREPARED 'global_x1';
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+COMMIT PREPARED 'global_x1';
+
+--ERROR case
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO t VALUES (1);
+COMMIT /*should do 2PC*/;

From 947ad43e20c296a613131b2a3136956709749336 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 14:12:17 +0500
Subject: [PATCH v21 1/5] Keep track of writing on non-temporary relation

---
 src/backend/executor/nodeModifyTable.c | 16 ++++++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 22 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 20a4c474cc..1ec07bad07 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -581,6 +581,10 @@ ExecInsert(ModifyTableState *mtstate,
 										   NULL,
 										   specToken);
 
+			/* Make note that we've wrote on non-temporary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -619,6 +623,10 @@ ExecInsert(ModifyTableState *mtstate,
 							   estate->es_output_cid,
 							   0, NULL);
 
+			/* Make note that we've wrote on non-temporary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -970,6 +978,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temporary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1482,6 +1494,10 @@ lreplace:;
 	if (canSetTag)
 		(estate->es_processed)++;
 
+	/* Make note that we've wrote on non-temporary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/* AFTER ROW UPDATE Triggers */
 	ExecARUpdateTriggers(estate, resultRelInfo, tupleid, oldtuple, slot,
 						 recheckIndexes,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7ee04babc2..a04fc70326 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -102,6 +102,12 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
-- 
2.23.0

From cd028035abd062c24bd3a9a719fbf4bf53733eb0 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Tue, 19 May 2020 15:06:38 +0900
Subject: [PATCH v21 3/5] Documentation update.

---
 doc/src/sgml/catalogs.sgml                | 145 +++++++++++++
 doc/src/sgml/config.sgml                  | 146 +++++++++++++
 doc/src/sgml/distributed-transaction.sgml | 154 ++++++++++++++
 doc/src/sgml/fdwhandler.sgml              | 236 ++++++++++++++++++++++
 doc/src/sgml/filelist.sgml                |   1 +
 doc/src/sgml/func.sgml                    |  89 ++++++++
 doc/src/sgml/monitoring.sgml              |  86 ++++++++
 doc/src/sgml/postgres.sgml                |   1 +
 doc/src/sgml/storage.sgml                 |   6 +
 9 files changed, 864 insertions(+)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index b1b077c97f..bdc60908be 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9216,6 +9216,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
       <entry>open cursors</entry>
      </row>
 
+     <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
      <row>
       <entry><link linkend="view-pg-file-settings"><structname>pg_file_settings</structname></link></entry>
       <entry>summary of configuration file contents</entry>
@@ -10931,6 +10936,146 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with that this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on that the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>initial</literal> : Initial status.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>resolved</literal> : This foreign transaction has been resolved.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-doubt status and
+       needs to be resolved by calling <function>pg_resolve_fdwxact</function>
+       function.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_prepared_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 9f2a4a2470..feef45e18c 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -9081,6 +9081,152 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether distributed transaction commits ensures that all
+         involved changes on foreign servers are committed or not. Valid
+         values are <literal>required</literal>, <literal>prefer</literal> and
+         <literal>disabled</literal>. The default setting is
+         <literal>disabled</literal>. Setting to <literal>disabled</literal>
+         don't use two-phase commit protocol to commit or rollback distributed
+         transactions. When set to <literal>required</literal> distributed
+         transactions strictly requires that all written servers can use
+         two-phase commit protocol.  That is, the distributed transaction cannot
+         commit if even one server does not support the prepare callback routine
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. In <literal>prefer</literal> and <literal>required</literal> case,
+         distributed transaction commit will wait for all involving foreign
+         transaction to be committed before the command return a "success"
+         indication to the client.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+
+        <note>
+         <para>
+          When <literal>disabled</literal> or <literal>prefer</literal> there
+          can be risk of database consistency among all servers that involved in
+          the distributed transaction when some foreign server crashes during
+          committing the distributed transaction.
+         </para>
+        </note>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If <literal>N</literal> local transactions each
+         across <literal>K</literal> foreign server this value need to be set
+         <literal>N * K</literal>, not just <literal>N</literal>.
+         This parameter can only be set at server start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000000..85d8e8e9e4
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,154 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Formerly, transactions on foreign server were not simply committed or rolled
+   back one by one. Therefore, when one foreign server had a problem during
+   commit, it was possible that transactions on only part of foreign servers
+   are committed while other transactions are rolled back. This used to leave
+   database data in an inconsistent state in term of federated database.
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatically,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).
+    A <productname>PostgreSQL</productname> server that received SQL is called
+    <firstterm>coordinator node</firstterm> who is responsible for coordinating
+    all the participanting transactions. Using two-phase commit protocol, the commit
+    sequence of distributed transaction performs with the following steps.
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers.
+      </para>
+     </listitem>
+    </orderedlist>
+
+   </para>
+
+   <para>
+    At the first step, <productname>PostgreSQL</productname> distributed
+    transaction manager prepares all transaction on the foreign servers if
+    two-phase commit is required. Two-phase commit is required when the
+    transaction modifies data on two or more servers including the local server
+    itself and <xref linkend="guc-foreign-twophase-commit"/>is
+    <literal>required</literal> or <literal>prefer</literal>. If all preparations
+    on foreign servers got successful go to the next step. Any failure happens
+    in this step <productname>PostgreSQL</productname> changes to rollback, then
+    rollback all transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the local commit step, <productname>PostgreSQL</productname> commit the
+    transaction locally. Any failure happens in this step
+    <productname>PostgreSQL</productname> changes rollback, then rollback all
+    transactions on both local and foreign servers.
+   </para>
+
+   <para>
+    At the final step, prepared transactions are resolved by a foreign transaction
+    resolver process.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>Manual Resolution of In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, distributed transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the coordinator node crashed during either preparing or
+    resolving distributed transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="pg-stat-foreign-xact-view"/>
+    view. These foreign transactions are resolved by foreign transaction resolver
+    process or executing <function>pg_resolve_foriegn_xact</function> function
+    manually.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolution">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that is
+    responsible for resolving both foreign transactions that are prepared by
+    online transactions and in-doubt transactions. They commit or rollback
+    prepared transaction on foreign servers if the coordinator received agreement
+    messages from all foreign servers during the first step.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on one database connecting to. On failure during resolution, they retry to
+    resolve at an interval of <varname>foreign_transaction_resolution_interval</varname>
+    time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped. So to drop the database, you can call
+     <function>pg_stop_foreign_xact_resovler</function> function to stop the
+     particular resolver process before dropping the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+   </para>
+
+   <para>
+    On the coordinator side, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that some extensions and parallel queries also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 6587678af2..dd0358ef22 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1415,6 +1415,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1894,4 +2015,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 68179f71cd..1ab8e80fdc 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 7c06afd3ea..e281bd33d8 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26126,6 +26126,95 @@ SELECT convert_from(pg_read_binary_file('file_in_utf8.txt'), 'UTF8');
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index acc6e2bc31..0df1073e4a 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -384,6 +384,14 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       </entry>
      </row>
 
+     <row>
+      <entry><structname>pg_stat_foreign_xact</structname><indexterm><primary>pg_stat_fdw_xact_resolver</primary></indexterm></entry>
+      <entry>One row per foreign transaction resolver process, showing statistics about
+       foreign transaction resolution. See <xref linkend="pg-stat-foreign-xact-view"/> for
+       details.
+      </entry>
+     </row>
+
     </tbody>
    </tgroup>
   </table>
@@ -1027,6 +1035,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       <entry><literal>CheckpointerMain</literal></entry>
       <entry>Waiting in main loop of checkpointer process.</entry>
      </row>
+     <row>
+      <entry><literal>FdwXactLauncherMain</literal></entry>
+      <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+     </row>
+     <row>
+      <entry><literal>FdwXactResolverMain</literal></entry>
+      <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+     </row>
+     <row>
+      <entry><literal>LogicalLauncherMain</literal></entry>
+      <entry>Waiting in main loop of logical launcher process.</entry>
+     </row>
      <row>
       <entry><literal>LogicalApplyMain</literal></entry>
       <entry>Waiting in main loop of logical replication apply process.</entry>
@@ -1248,6 +1268,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       <entry><literal>DataFileWrite</literal></entry>
       <entry>Waiting for a write to a relation data file.</entry>
      </row>
+     <row>
+      <entry><literal>FdwXactFileRead</literal></entry>
+      <entry>Waiting for a read of a foreign transaction state file.</entry>
+     </row>
+     <row>
+      <entry><literal>FdwXactFileSync</literal></entry>
+      <entry>Waiting for a foreign transaction state file to reach stable storage.</entry>
+     </row>
+     <row>
+      <entry><literal>FdwXactFileWrite</literal></entry>
+      <entry>Waiting for a write of a foreign transaction state file.</entry>
+     </row>
      <row>
       <entry><literal>LockFileAddToDataDirRead</literal></entry>
       <entry>Waiting for a read while adding a line to the data directory lock
@@ -1525,6 +1557,11 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       <entry>Waiting for activity from a child process while
        executing a <literal>Gather</literal> plan node.</entry>
      </row>
+     <row>
+      <entry><literal>FdwXactResolution</literal></entry>
+      <entry>Waiting for all foreign transaction participants to be resolved during
+       atomic commit among foreign servers.</entry>
+     </row>
      <row>
       <entry><literal>HashBatchAllocate</literal></entry>
       <entry>Waiting for an elected Parallel Hash participant to allocate a hash
@@ -1836,6 +1873,19 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       <entry>Waiting to read or update dynamic shared memory allocation
        information.</entry>
      </row>
+     <row>
+      <entry><literal>FdwXactLock</literal></entry>
+      <entry>Waiting to read or update the state of foreign transactions.</entry>
+     </row>
+     <row>
+      <entry><literal>FdwXactResolutionLock</literal></entry>
+      <entry>Waiting to read or update information of foreign trasnaction
+       resolution.</entry>
+     </row>
+     <row>
+      <entry><literal>LogicalRepWorkerLock</literal></entry>
+      <entry>Waiting for action on logical replication worker to finish.</entry>
+     </row>
      <row>
       <entry><literal>LockFastPath</literal></entry>
       <entry>Waiting to read or update a process' fast-path lock
@@ -2971,6 +3021,42 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i
    connection.
   </para>
 
+  <table id="pg-stat-foreign-xact-view" xreflabel="pg_stat_foreign_xact">
+   <title><structname>pg_stat_foreign_xact</structname> View</title>
+   <tgroup cols="3">
+    <thead>
+    <row>
+      <entry>Column</entry>
+      <entry>Type</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+
+   <tbody>
+    <row>
+     <entry><structfield>pid</structfield></entry>
+     <entry><type>integer</type></entry>
+     <entry>Process ID of a foreign transaction resolver process</entry>
+    </row>
+    <row>
+     <entry><structfield>dbid</structfield></entry>
+     <entry><type>oid</type></entry>
+     <entry>OID of the database to which the foreign transaction resolver is connected</entry>
+    </row>
+    <row>
+     <entry><structfield>last_resolved_time</structfield></entry>
+     <entry><type>timestamp with time zone</type></entry>
+     <entry>Time at which the process last resolved a foreign transaction</entry>
+    </row>
+   </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   The <structname>pg_stat_fdw_xact_resolver</structname> view will contain one
+   row per foreign transaction resolver process, showing state of resolution
+   of foreign transactions.
+  </para>
 
   <table id="pg-stat-archiver-view" xreflabel="pg_stat_archiver">
    <title><structname>pg_stat_archiver</structname> View</title>
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index c41ce9499b..5ef1f4a329 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -170,6 +170,7 @@ break is not needed in a wider output rendering.
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index ea08d0b614..58f1e4fd15 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -83,6 +83,12 @@ Item
   subsystem</entry>
 </row>
 
+<row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
 <row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
-- 
2.23.0

From 42b1a1bbef2fd7eb928becc183cfac865d901276 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 21:41:29 +0500
Subject: [PATCH v21 5/5] Add regression tests for atomic commit.

---
 src/test/modules/Makefile                     |   1 +
 src/test/modules/test_fdwxact/.gitignore      |   4 +
 src/test/modules/test_fdwxact/Makefile        |  28 ++
 .../test_fdwxact/expected/test_fdwxact.out    | 233 +++++++++
 src/test/modules/test_fdwxact/fdwxact.conf    |   7 +
 .../modules/test_fdwxact/sql/test_fdwxact.sql | 205 ++++++++
 src/test/modules/test_fdwxact/t/001_basic.pl  | 137 +++++
 .../test_fdwxact/test_fdwxact--1.0.sql        |  44 ++
 src/test/modules/test_fdwxact/test_fdwxact.c  | 471 ++++++++++++++++++
 .../modules/test_fdwxact/test_fdwxact.control |   4 +
 src/test/recovery/Makefile                    |   2 +-
 src/test/recovery/t/021_fdwxact.pl            | 175 +++++++
 src/test/regress/pg_regress.c                 |  13 +-
 13 files changed, 1319 insertions(+), 5 deletions(-)
 create mode 100644 src/test/modules/test_fdwxact/.gitignore
 create mode 100644 src/test/modules/test_fdwxact/Makefile
 create mode 100644 src/test/modules/test_fdwxact/expected/test_fdwxact.out
 create mode 100644 src/test/modules/test_fdwxact/fdwxact.conf
 create mode 100644 src/test/modules/test_fdwxact/sql/test_fdwxact.sql
 create mode 100644 src/test/modules/test_fdwxact/t/001_basic.pl
 create mode 100644 src/test/modules/test_fdwxact/test_fdwxact--1.0.sql
 create mode 100644 src/test/modules/test_fdwxact/test_fdwxact.c
 create mode 100644 src/test/modules/test_fdwxact/test_fdwxact.control
 create mode 100644 src/test/recovery/t/021_fdwxact.pl

diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 29de73c060..8a48e6ba19 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -13,6 +13,7 @@ SUBDIRS = \
 		  test_bloomfilter \
 		  test_ddl_deparse \
 		  test_extensions \
+		  test_fdwxact \
 		  test_ginpostinglist \
 		  test_integerset \
 		  test_misc \
diff --git a/src/test/modules/test_fdwxact/.gitignore b/src/test/modules/test_fdwxact/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/src/test/modules/test_fdwxact/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/test_fdwxact/Makefile b/src/test/modules/test_fdwxact/Makefile
new file mode 100644
index 0000000000..b3fc99aee3
--- /dev/null
+++ b/src/test/modules/test_fdwxact/Makefile
@@ -0,0 +1,28 @@
+# src/test/modules/test_fdwxact/Makefile
+
+MODULE_big = test_fdwxact
+OBJS = \
+	$(WIN32RES) \
+	test_fdwxact.o
+PGFILEDESC = "test_fdwxact - test code for src/backend/access/fdwxact"
+
+EXTENSION = test_fdwxact
+DATA = test_fdwxact--1.0.sql
+
+REGRESS_OPTS = --temp-config=$(top_srcdir)/src/test/modules/test_fdwxact/fdwxact.conf
+REGRESS = test_fdwxact
+
+NO_INSTALLCHECK = 1
+
+TAP_TESTS =1
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_fdwxact/expected/test_fdwxact.out b/src/test/modules/test_fdwxact/expected/test_fdwxact.out
new file mode 100644
index 0000000000..ab4ff2d89a
--- /dev/null
+++ b/src/test/modules/test_fdwxact/expected/test_fdwxact.out
@@ -0,0 +1,233 @@
+--
+-- Test for foreign transaction management.
+--
+CREATE EXTENSION test_fdwxact;
+-- setup two servers that don't support transaction management API
+CREATE SERVER srv_1 FOREIGN DATA WRAPPER test_fdw;
+CREATE SERVER srv_2 FOREIGN DATA WRAPPER test_fdw;
+-- setup two servers that support only commit and rollback API
+CREATE SERVER srv_no2pc_1 FOREIGN DATA WRAPPER test_no2pc_fdw;
+CREATE SERVER srv_no2pc_2 FOREIGN DATA WRAPPER test_no2pc_fdw;
+-- setup two servers that support commit, rollback and prepare API.
+-- That is, those two server support two-phase commit protocol.
+CREATE SERVER srv_2pc_1 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE SERVER srv_2pc_2 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE TABLE t (i int);
+CREATE FOREIGN TABLE ft_1 (i int) SERVER srv_1;
+CREATE FOREIGN TABLE ft_2 (i int) SERVER srv_2;
+CREATE FOREIGN TABLE ft_no2pc_1 (i int) SERVER srv_no2pc_1;
+CREATE FOREIGN TABLE ft_no2pc_2 (i int) SERVER srv_no2pc_2;
+CREATE FOREIGN TABLE ft_2pc_1 (i int) SERVER srv_2pc_1;
+CREATE FOREIGN TABLE ft_2pc_2 (i int) SERVER srv_2pc_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_2;
+-- Test 'disabled' case.
+-- Modifies one or two servers but since we don't require two-phase
+-- commit, all case should not raise an error.
+SET foreign_twophase_commit TO disabled;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+ROLLBACK;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+ROLLBACK;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+ROLLBACK;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+-- Test 'required' case.
+-- In this case, when two-phase commit is required, all servers
+-- which are involved in the and modified need to support two-phase
+-- commit protocol. Otherwise transaction will rollback.
+SET foreign_twophase_commit TO 'required';
+-- Ok. Writing only one server doesn't require two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+-- Ok. Writing two servers, we require two-phase commit and success.
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+-- Ok. Only reading servers doesn't require two-phase commit.
+BEGIN;
+SELECT * FROM ft_2pc_1;
+ i 
+---
+(0 rows)
+
+SELECT * FROM ft_2pc_2;
+ i 
+---
+(0 rows)
+
+COMMIT;
+BEGIN;
+SELECT * FROM ft_1;
+ i 
+---
+(0 rows)
+
+SELECT * FROM ft_no2pc_1;
+ i 
+---
+(0 rows)
+
+COMMIT;
+-- Ok. Read one server and write one server.
+BEGIN;
+SELECT * FROM ft_1;
+ i 
+---
+(0 rows)
+
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+SELECT * FROM ft_no2pc_1;
+ i 
+---
+(0 rows)
+
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+-- Error. ft_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. ft_no2pc_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. Both ft_1 and ft_2 don't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. Both ft_no2pc_1 and ft_no2pc_2 don't support two-phase
+-- commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. Two-phase commit is required because of writes on two
+-- servers: local node and ft_no2pc_1. But ft_no2pc_1 doesn't support
+-- two-phase commit.
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Test 'prefer' case.
+-- The cases where failed in 'required' case shoul pass in 'prefer'.
+-- We simply commit/rollback a transaction in one-phase on a server
+-- that doesn't support two-phase commit, instead of error.
+SET foreign_twophase_commit TO 'prefer';
+-- We modify at least one server that doesn't support two-phase commit.
+-- These servers are committed in one-phase.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+-- Test cases for preparing the local transaction.
+SET foreign_twophase_commit TO 'prefer';
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+INSERT INTO t VALUES(3);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+ count 
+-------
+     1
+(1 row)
+
+COMMIT PREPARED 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+ count 
+-------
+     0
+(1 row)
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+ count 
+-------
+     1
+(1 row)
+
+COMMIT PREPARED 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+ count 
+-------
+     0
+(1 row)
+
+-- PREPARE needs all involved foreign servers to support two-phsae
+-- commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+ERROR:  cannot PREPARE a distributed transaction that has operated on a foreign server
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+ERROR:  cannot PREPARE a distributed transaction that has operated on a foreign server
diff --git a/src/test/modules/test_fdwxact/fdwxact.conf b/src/test/modules/test_fdwxact/fdwxact.conf
new file mode 100644
index 0000000000..20e4a671df
--- /dev/null
+++ b/src/test/modules/test_fdwxact/fdwxact.conf
@@ -0,0 +1,7 @@
+shared_preload_libraries = 'test_fdwxact'
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 1
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = disabled
diff --git a/src/test/modules/test_fdwxact/sql/test_fdwxact.sql b/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
new file mode 100644
index 0000000000..358eea8630
--- /dev/null
+++ b/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
@@ -0,0 +1,205 @@
+--
+-- Test for foreign transaction management.
+--
+
+CREATE EXTENSION test_fdwxact;
+
+-- setup two servers that don't support transaction management API
+CREATE SERVER srv_1 FOREIGN DATA WRAPPER test_fdw;
+CREATE SERVER srv_2 FOREIGN DATA WRAPPER test_fdw;
+
+-- setup two servers that support only commit and rollback API
+CREATE SERVER srv_no2pc_1 FOREIGN DATA WRAPPER test_no2pc_fdw;
+CREATE SERVER srv_no2pc_2 FOREIGN DATA WRAPPER test_no2pc_fdw;
+
+-- setup two servers that support commit, rollback and prepare API.
+-- That is, those two server support two-phase commit protocol.
+CREATE SERVER srv_2pc_1 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE SERVER srv_2pc_2 FOREIGN DATA WRAPPER test_2pc_fdw;
+
+CREATE TABLE t (i int);
+CREATE FOREIGN TABLE ft_1 (i int) SERVER srv_1;
+CREATE FOREIGN TABLE ft_2 (i int) SERVER srv_2;
+CREATE FOREIGN TABLE ft_no2pc_1 (i int) SERVER srv_no2pc_1;
+CREATE FOREIGN TABLE ft_no2pc_2 (i int) SERVER srv_no2pc_2;
+CREATE FOREIGN TABLE ft_2pc_1 (i int) SERVER srv_2pc_1;
+CREATE FOREIGN TABLE ft_2pc_2 (i int) SERVER srv_2pc_2;
+
+CREATE USER MAPPING FOR PUBLIC SERVER srv_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_2;
+
+-- Test 'disabled' case.
+-- Modifies one or two servers but since we don't require two-phase
+-- commit, all case should not raise an error.
+SET foreign_twophase_commit TO disabled;
+
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+ROLLBACK;
+
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+ROLLBACK;
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+ROLLBACK;
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+
+-- Test 'required' case.
+-- In this case, when two-phase commit is required, all servers
+-- which are involved in the and modified need to support two-phase
+-- commit protocol. Otherwise transaction will rollback.
+SET foreign_twophase_commit TO 'required';
+
+-- Ok. Writing only one server doesn't require two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Ok. Writing two servers, we require two-phase commit and success.
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Ok. Only reading servers doesn't require two-phase commit.
+BEGIN;
+SELECT * FROM ft_2pc_1;
+SELECT * FROM ft_2pc_2;
+COMMIT;
+BEGIN;
+SELECT * FROM ft_1;
+SELECT * FROM ft_no2pc_1;
+COMMIT;
+
+-- Ok. Read one server and write one server.
+BEGIN;
+SELECT * FROM ft_1;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+SELECT * FROM ft_no2pc_1;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Error. ft_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Error. ft_no2pc_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Error. Both ft_1 and ft_2 don't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+
+-- Error. Both ft_no2pc_1 and ft_no2pc_2 don't support two-phase
+-- commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+
+-- Error. Two-phase commit is required because of writes on two
+-- servers: local node and ft_no2pc_1. But ft_no2pc_1 doesn't support
+-- two-phase commit.
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+
+-- Test 'prefer' case.
+-- The cases where failed in 'required' case shoul pass in 'prefer'.
+-- We simply commit/rollback a transaction in one-phase on a server
+-- that doesn't support two-phase commit, instead of error.
+SET foreign_twophase_commit TO 'prefer';
+
+-- We modify at least one server that doesn't support two-phase commit.
+-- These servers are committed in one-phase.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+
+-- Test cases for preparing the local transaction.
+SET foreign_twophase_commit TO 'prefer';
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+INSERT INTO t VALUES(3);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+COMMIT PREPARED 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+COMMIT PREPARED 'global_x1';
+SELECT count(*)  FROM pg_foreign_xacts();
+
+-- PREPARE needs all involved foreign servers to support two-phsae
+-- commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
diff --git a/src/test/modules/test_fdwxact/t/001_basic.pl b/src/test/modules/test_fdwxact/t/001_basic.pl
new file mode 100644
index 0000000000..8d48a74e86
--- /dev/null
+++ b/src/test/modules/test_fdwxact/t/001_basic.pl
@@ -0,0 +1,137 @@
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 11;
+
+my $node = get_new_node('main');
+$node->init;
+$node->append_conf('postgresql.conf', qq(
+shared_preload_libraries = 'test_fdwxact'
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = required
+test_fdwxact.log_api_calls = true
+				   ));
+$node->start;
+
+$node->psql(
+	'postgres', "
+CREATE EXTENSION test_fdwxact;
+CREATE SERVER srv FOREIGN DATA WRAPPER test_fdw;
+CREATE SERVER srv_no2pc FOREIGN DATA WRAPPER test_no2pc_fdw;
+CREATE SERVER srv_2pc_1 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE SERVER srv_2pc_2 FOREIGN DATA WRAPPER test_2pc_fdw;
+
+CREATE TABLE t (i int);
+CREATE FOREIGN TABLE ft (i int) SERVER srv;
+CREATE FOREIGN TABLE ft_no2pc (i int) SERVER srv_no2pc;
+CREATE FOREIGN TABLE ft_2pc_1 (i int) SERVER srv_2pc_1;
+CREATE FOREIGN TABLE ft_2pc_2 (i int) SERVER srv_2pc_2;
+
+CREATE USER MAPPING FOR PUBLIC SERVER srv;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_2;
+	");
+
+sub run_transaction
+{
+	my ($node, $prepsql, $sql, $endsql) = @_;
+
+	$endsql = 'COMMIT' unless defined $endsql;
+
+	local $ENV{PGHOST} = $node->host;
+	local $ENV{PGPORT} = $node->port;
+
+	truncate $node->logfile, 0;
+
+	$node->safe_psql('postgres', $prepsql);
+	my ($cmdret, $stdout, $stderr) = $node->psql('postgres',
+												 "BEGIN;
+												 SELECT txid_current() as xid;
+												 $sql
+												 $endsql;
+												 ");
+	my $log = TestLib::slurp_file($node->logfile);
+
+	return $log, $stdout;
+}
+
+my ($log, $xid);
+
+# The transaction is committed using two-phase commit.
+($log, $xid) = run_transaction($node, "",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+like($log, qr/commit prepared tx_$xid on srv_2pc_1/, "commit prepared transaction-1");
+like($log, qr/commit prepared tx_$xid on srv_2pc_2/, "commit prepared transaction-2");
+
+# Similary, two-phase commit is used.
+($log, $xid) = run_transaction($node, "",
+					  "INSERT INTO t VALUES(1);
+					  INSERT INTO ft_2pc_1 VALUES(1);");
+like($log, qr/commit prepared tx_$xid on srv_2pc_1/, "commit prepared transaction-3");
+
+# Test the case where transaction attempting prepare the local transaction fails after
+# preparing foreign transactions. The first attempt should be succeeded, but the second
+# attempt will fail after preparing foreign transaction, and should rollback the prepared
+# foreign transaction.
+($log, $xid) = run_transaction($node, "",
+							   "INSERT INTO t VALUES(1);
+							   INSERT INTO ft_2pc_1 VALUES(1);",
+							   "PREPARE TRANSACTION 'tx1'");
+($log, $xid) = run_transaction($node, "",
+							   "INSERT INTO t VALUES(1);
+							   INSERT INTO ft_2pc_1 VALUES(1);",
+							   "PREPARE TRANSACTION 'tx1'");
+like($log, qr/rollback prepared tx_$xid on srv_2pc_1/, "failure after prepare transaction");
+$node->safe_psql('postgres', "COMMIT PREPARED 'tx1'");
+
+# Inject an error into prepare phase on srv_2pc_1. The transaction fails during
+# preparing the foreign transaction on srv_2pc_1. Then, we try to both 'rollback' and
+# 'rollback prepared' the foreign transaction, and rollback another foreign
+# transaction.
+($log, $xid) = run_transaction($node,
+							   "SELECT test_inject_error('error', 'prepare', 'srv_2pc_1');",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+like($log, qr/rollback $xid on srv_2pc_1/, "rollback on failed server");
+like($log, qr/rollback prepared tx_$xid on srv_2pc_1/, "rollback prepared on failed server");
+like($log, qr/rollback $xid on srv_2pc_2/, "rollback on another server");
+
+# Inject an panic into prepare phase on srv_2pc_2. The server crashes after preparing both
+# foreign transaction. After the restart, those transactions are recovered as in-doubt
+# transactions. We check if the resolver process rollbacks those transaction after recovery.
+($log, $xid) = run_transaction($node,
+							   "SELECT test_inject_error('panic', 'prepare', 'srv_2pc_2');",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+$node->restart();
+$node->poll_query_until('postgres',
+						"SELECT count(*) = 0 FROM pg_foreign_xacts")
+  or die "Timeout while waiting for resolver process to resolve in-doubt transactions";
+$log = TestLib::slurp_file($node->logfile);
+like($log, qr/rollback prepared tx_[0-9]+ on srv_2pc_1/, "resolver rolled back in-doubt transaction");
+like($log, qr/rollback prepared tx_[0-9]+ on srv_2pc_2/, "resolver rolled back in-doubt transaction");
+truncate $node->logfile, 0;
+
+# Inject an panic into commit phase on srv_2pc_1. The server crashes due to the panic
+# error raised by resolver process during commit prepared foreign transaction on srv_2pc_1.
+# After the restart, those transactions are recovered as in-doubt transactions. We check if
+# the resolver process commits those transaction after recovery.
+($log, $xid) = run_transaction($node,
+							   "SELECT test_inject_error('panic', 'commit', 'srv_2pc_1');",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+$node->restart();
+$node->poll_query_until('postgres',
+						"SELECT count(*) = 0 FROM pg_foreign_xacts")
+  or die "Timeout while waiting for resolver process to resolve in-doubt transactions";
+$log = TestLib::slurp_file($node->logfile);
+like($log, qr/commit prepared tx_[0-9]+ on srv_2pc_1/, "resolver rolled back in-doubt transaction");
+like($log, qr/commit prepared tx_[0-9]+ on srv_2pc_2/, "resolver rolled back in-doubt transaction");
+truncate $node->logfile, 0;
diff --git a/src/test/modules/test_fdwxact/test_fdwxact--1.0.sql b/src/test/modules/test_fdwxact/test_fdwxact--1.0.sql
new file mode 100644
index 0000000000..f676dfe04b
--- /dev/null
+++ b/src/test/modules/test_fdwxact/test_fdwxact--1.0.sql
@@ -0,0 +1,44 @@
+/* src/test/modules/test_atomic_commit/test_atomic_commit--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_atomic_commit" to load this file. \quit
+
+-- test_fdw doesn't use transaction API
+CREATE FUNCTION test_fdw_handler()
+RETURNS fdw_handler
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FOREIGN DATA WRAPPER test_fdw
+  HANDLER test_fdw_handler;
+
+-- test_no2pc_fdw uses only COMMIT and ROLLBACK API
+CREATE FUNCTION test_no2pc_fdw_handler()
+RETURNS fdw_handler
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FOREIGN DATA WRAPPER test_no2pc_fdw
+  HANDLER test_no2pc_fdw_handler;
+
+-- test_2pc uses PREPARE API as well
+CREATE FUNCTION test_2pc_fdw_handler()
+RETURNS fdw_handler
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FOREIGN DATA WRAPPER test_2pc_fdw
+  HANDLER test_2pc_fdw_handler;
+
+CREATE FUNCTION test_inject_error(
+elevel text,
+phase text,
+server text)
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION test_reset_error()
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
diff --git a/src/test/modules/test_fdwxact/test_fdwxact.c b/src/test/modules/test_fdwxact/test_fdwxact.c
new file mode 100644
index 0000000000..a75d3cde14
--- /dev/null
+++ b/src/test/modules/test_fdwxact/test_fdwxact.c
@@ -0,0 +1,471 @@
+/*-------------------------------------------------------------------------
+ *
+ * test_fdwxact.c
+ *		  Test modules for foreign transaction management
+ *
+ * This module implements three types of foreign data wrapper: the first
+ * doesn't support any transaction FDW APIs, the second supports only
+ * commit and rollback API and the third supports all transaction API including
+ * prepare.
+ *
+ * Also, this module has an ability to inject an error at prepare callback or
+ * commit callback using test_inject_error() SQL function. The information of
+ * injected error is stored in the shared memory so that backend processes and
+ * resolver processes can see it.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		src/test/modules/test_fdwxact/test_fdwxact.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact.h"
+#include "commands/defrem.h"
+#include "access/reloptions.h"
+#include "foreign/fdwapi.h"
+#include "funcapi.h"
+#include "miscadmin.h"
+#include "optimizer/clauses.h"
+#include "optimizer/cost.h"
+#include "optimizer/optimizer.h"
+#include "optimizer/pathnode.h"
+#include "optimizer/paths.h"
+#include "optimizer/planmain.h"
+#include "optimizer/restrictinfo.h"
+#include "storage/ipc.h"
+#include "storage/spin.h"
+#include "utils/builtins.h"
+
+PG_MODULE_MAGIC;
+
+#define TEST_FDWXCT_MAX_NAME_LEN 32
+
+typedef struct testFdwXactSharedState
+{
+	char	elevel[TEST_FDWXCT_MAX_NAME_LEN];
+	char	phase[TEST_FDWXCT_MAX_NAME_LEN];
+	char	server[TEST_FDWXCT_MAX_NAME_LEN];
+	LWLock	*lock;
+} testFdwXactSharedState;
+testFdwXactSharedState *fxss = NULL;
+
+static shmem_startup_hook_type prev_shmem_startup_hook = NULL;
+static bool log_api_calls = false;
+
+void _PG_init(void);
+void _PG_fini(void);
+PG_FUNCTION_INFO_V1(test_fdw_handler);
+PG_FUNCTION_INFO_V1(test_no2pc_fdw_handler);
+PG_FUNCTION_INFO_V1(test_2pc_fdw_handler);
+PG_FUNCTION_INFO_V1(test_inject_error);
+PG_FUNCTION_INFO_V1(test_reset_error);
+
+static void test_fdwxact_shmem_startup(void);
+static bool check_event(char *servername, char *phase, int *elevel);
+static void testGetForeignRelSize(PlannerInfo *root,
+								  RelOptInfo *baserel,
+								  Oid foreigntableid);
+static void testGetForeignPaths(PlannerInfo *root,
+								RelOptInfo *baserel,
+								Oid foreigntableid);
+static ForeignScan *testGetForeignPlan(PlannerInfo *root,
+									   RelOptInfo *foreignrel,
+									   Oid foreigntableid,
+									   ForeignPath *best_path,
+									   List *tlist,
+									   List *scan_clauses,
+									   Plan *outer_plan);
+static void testBeginForeignScan(ForeignScanState *node, int eflags);
+static TupleTableSlot *testIterateForeignScan(ForeignScanState *node);
+static void testReScanForeignScan(ForeignScanState *node);
+static void testEndForeignScan(ForeignScanState *node);
+static void testBeginForeignModify(ModifyTableState *mtstate,
+								   ResultRelInfo *resultRelInfo,
+								   List *fdw_private,
+								   int subplan_index,
+								   int eflags);
+static TupleTableSlot *testExecForeignInsert(EState *estate,
+											 ResultRelInfo *resultRelInfo,
+											 TupleTableSlot *slot,
+											 TupleTableSlot *planSlot);
+static void testEndForeignModify(EState *estate,
+								 ResultRelInfo *resultRelInfo);
+static void testBeginForeignInsert(ModifyTableState *mtstate,
+								   ResultRelInfo *resultRelInfo);
+static void testEndForeignInsert(EState *estate,
+								 ResultRelInfo *resultRelInfo);
+static int	testIsForeignRelUpdatable(Relation rel);
+static void testPrepareForeignTransaction(FdwXactRslvState *state);
+static void testCommitForeignTransaction(FdwXactRslvState *state);
+static void testRollbackForeignTransaction(FdwXactRslvState *state);
+static char *testGetPrepareId(TransactionId xid, Oid serverid,
+							  Oid userid, int *prep_id_len);
+
+void
+_PG_init(void)
+{
+	DefineCustomBoolVariable("test_fdwxact.log_api_calls",
+							 "Report transaction API calls to logs.",
+							 NULL,
+							 &log_api_calls,
+							 false,
+							 PGC_USERSET,
+							 0,
+							 NULL, NULL, NULL);
+
+	RequestAddinShmemSpace(MAXALIGN(sizeof(testFdwXactSharedState)));
+	RequestNamedLWLockTranche("test_fdwxact", 1);
+
+	prev_shmem_startup_hook = shmem_startup_hook;
+	shmem_startup_hook = test_fdwxact_shmem_startup;
+}
+
+void
+_PG_fini(void)
+{
+	/* Uninstall hooks. */
+	shmem_startup_hook = prev_shmem_startup_hook;
+}
+
+static void
+test_fdwxact_shmem_startup(void)
+{
+	bool found;
+
+	if (prev_shmem_startup_hook)
+		prev_shmem_startup_hook();
+
+	fxss = ShmemInitStruct("test_fdwxact",
+						   sizeof(testFdwXactSharedState),
+						   &found);
+	if (!found)
+	{
+		memset(fxss->elevel, 0, TEST_FDWXCT_MAX_NAME_LEN);
+		memset(fxss->phase, 0, TEST_FDWXCT_MAX_NAME_LEN);
+		memset(fxss->server, 0, TEST_FDWXCT_MAX_NAME_LEN);
+		fxss->lock = &(GetNamedLWLockTranche("test_fdwxact"))->lock;
+	}
+}
+
+Datum
+test_fdw_handler(PG_FUNCTION_ARGS)
+{
+	FdwRoutine *routine = makeNode(FdwRoutine);
+
+	/* Functions for scanning foreign tables */
+	routine->GetForeignRelSize = testGetForeignRelSize;
+	routine->GetForeignPaths = testGetForeignPaths;
+	routine->GetForeignPlan = testGetForeignPlan;
+	routine->BeginForeignScan = testBeginForeignScan;
+	routine->IterateForeignScan = testIterateForeignScan;
+	routine->ReScanForeignScan = testReScanForeignScan;
+	routine->EndForeignScan = testEndForeignScan;
+
+	/* Functions for updating foreign tables */
+	routine->AddForeignUpdateTargets = NULL;
+	routine->PlanForeignModify = NULL;
+	routine->BeginForeignModify = testBeginForeignModify;
+	routine->ExecForeignInsert = testExecForeignInsert;
+	routine->EndForeignModify = testEndForeignModify;
+	routine->BeginForeignInsert = testBeginForeignInsert;
+	routine->EndForeignInsert = testEndForeignInsert;
+	routine->IsForeignRelUpdatable = testIsForeignRelUpdatable;
+
+	PG_RETURN_POINTER(routine);
+}
+
+Datum
+test_no2pc_fdw_handler(PG_FUNCTION_ARGS)
+{
+	FdwRoutine *routine = makeNode(FdwRoutine);
+
+	/* Functions for scanning foreign tables */
+	routine->GetForeignRelSize = testGetForeignRelSize;
+	routine->GetForeignPaths = testGetForeignPaths;
+	routine->GetForeignPlan = testGetForeignPlan;
+	routine->BeginForeignScan = testBeginForeignScan;
+	routine->IterateForeignScan = testIterateForeignScan;
+	routine->ReScanForeignScan = testReScanForeignScan;
+	routine->EndForeignScan = testEndForeignScan;
+
+	/* Functions for updating foreign tables */
+	routine->AddForeignUpdateTargets = NULL;
+	routine->PlanForeignModify = NULL;
+	routine->BeginForeignModify = testBeginForeignModify;
+	routine->ExecForeignInsert = testExecForeignInsert;
+	routine->EndForeignModify = testEndForeignModify;
+	routine->BeginForeignInsert = testBeginForeignInsert;
+	routine->EndForeignInsert = testEndForeignInsert;
+	routine->IsForeignRelUpdatable = testIsForeignRelUpdatable;
+
+	/* Support only COMMIT and ROLLBACK */
+	routine->CommitForeignTransaction = testCommitForeignTransaction;
+	routine->RollbackForeignTransaction = testRollbackForeignTransaction;
+
+	PG_RETURN_POINTER(routine);
+}
+
+Datum
+test_2pc_fdw_handler(PG_FUNCTION_ARGS)
+{
+	FdwRoutine *routine = makeNode(FdwRoutine);
+
+	/* Functions for scanning foreign tables */
+	routine->GetForeignRelSize = testGetForeignRelSize;
+	routine->GetForeignPaths = testGetForeignPaths;
+	routine->GetForeignPlan = testGetForeignPlan;
+	routine->BeginForeignScan = testBeginForeignScan;
+	routine->IterateForeignScan = testIterateForeignScan;
+	routine->ReScanForeignScan = testReScanForeignScan;
+	routine->EndForeignScan = testEndForeignScan;
+
+	/* Functions for updating foreign tables */
+	routine->AddForeignUpdateTargets = NULL;
+	routine->PlanForeignModify = NULL;
+	routine->BeginForeignModify = testBeginForeignModify;
+	routine->ExecForeignInsert = testExecForeignInsert;
+	routine->EndForeignModify = testEndForeignModify;
+	routine->BeginForeignInsert = testBeginForeignInsert;
+	routine->EndForeignInsert = testEndForeignInsert;
+	routine->IsForeignRelUpdatable = testIsForeignRelUpdatable;
+
+	/* Support all functions for foreign transactions */
+	routine->GetPrepareId = testGetPrepareId;
+	routine->PrepareForeignTransaction = testPrepareForeignTransaction;
+	routine->CommitForeignTransaction = testCommitForeignTransaction;
+	routine->RollbackForeignTransaction = testRollbackForeignTransaction;
+
+	PG_RETURN_POINTER(routine);
+}
+
+static void
+testGetForeignRelSize(PlannerInfo *root,
+					  RelOptInfo *baserel,
+					  Oid foreigntableid)
+{
+	baserel->pages = 10;
+	baserel->tuples = 100;
+}
+
+static void
+testGetForeignPaths(PlannerInfo *root,
+					RelOptInfo *baserel,
+					Oid foreigntableid)
+{
+	add_path(baserel, (Path *) create_foreignscan_path(root, baserel,
+													   NULL,
+													   10, 10, 10,
+													   NIL,
+													   baserel->lateral_relids,
+													   NULL, NIL));
+}
+
+static ForeignScan *
+testGetForeignPlan(PlannerInfo *root,
+				   RelOptInfo *foreignrel,
+				   Oid foreigntableid,
+				   ForeignPath *best_path,
+				   List *tlist,
+				   List *scan_clauses,
+				   Plan *outer_plan)
+{
+	return make_foreignscan(tlist,
+							NIL,
+							foreignrel->relid,
+							NIL,
+							NULL,
+							NIL,
+							NIL,
+							outer_plan);
+}
+
+static void
+testBeginForeignScan(ForeignScanState *node, int eflags)
+{
+	return;
+}
+
+static TupleTableSlot *
+testIterateForeignScan(ForeignScanState *node)
+{
+	return ExecClearTuple(node->ss.ss_ScanTupleSlot);
+}
+
+static void
+testReScanForeignScan(ForeignScanState *node)
+{
+	return;
+}
+
+static void
+testEndForeignScan(ForeignScanState *node)
+{
+	return;
+}
+
+static void
+testBeginForeignModify(ModifyTableState *mtstate,
+					   ResultRelInfo *resultRelInfo,
+					   List *fdw_private,
+					   int subplan_index,
+					   int eflags)
+{
+	return;
+}
+
+static TupleTableSlot *
+testExecForeignInsert(EState *estate,
+					  ResultRelInfo *resultRelInfo,
+					  TupleTableSlot *slot,
+					  TupleTableSlot *planSlot)
+{
+	return slot;
+}
+
+static void
+testEndForeignModify(EState *estate,
+					 ResultRelInfo *resultRelInfo)
+{
+	return;
+}
+
+static void
+testBeginForeignInsert(ModifyTableState *mtstate,
+					   ResultRelInfo *resultRelInfo)
+{
+	return;
+}
+
+static void
+testEndForeignInsert(EState *estate,
+					 ResultRelInfo *resultRelInfo)
+{
+	return;
+}
+
+static int
+testIsForeignRelUpdatable(Relation rel)
+{
+	/* allow only inserts */
+	return (1 << CMD_INSERT);
+}
+
+static char *
+testGetPrepareId(TransactionId xid, Oid serverid,
+				 Oid userid, int *prep_id_len)
+{
+	static char buf[32] = {0};
+
+	*prep_id_len = snprintf(buf, 32, "tx_%u", xid);
+
+	return buf;
+}
+
+static void
+testPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	int elevel;
+
+	if (check_event(state->server->servername, "prepare", &elevel))
+		elog(elevel, "injected error at prepare");
+
+	if (log_api_calls)
+		ereport(LOG, (errmsg("prepare %s on %s",
+							 state->fdwxact_id,
+							 state->server->servername)));
+}
+
+static void
+testCommitForeignTransaction(FdwXactRslvState *state)
+{
+	int elevel;
+
+	if (check_event(state->server->servername, "commit", &elevel))
+		elog(elevel, "injected error at commit");
+
+	if (log_api_calls)
+	{
+		if (state->flags && FDWXACT_FLAG_ONEPHASE)
+			ereport(LOG, (errmsg("commit %u on %s",
+								 state->xid, state->server->servername)));
+		else
+			ereport(LOG, (errmsg("commit prepared %s on %s",
+								 state->fdwxact_id,
+								 state->server->servername)));
+	}
+}
+
+static void
+testRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	if (log_api_calls)
+	{
+		if (state->flags && FDWXACT_FLAG_ONEPHASE)
+			ereport(LOG, (errmsg("rollback %u on %s",
+								 state->xid, state->server->servername)));
+		else
+			ereport(LOG, (errmsg("rollback prepared %s on %s",
+								 state->fdwxact_id,
+								 state->server->servername)));
+	}
+}
+
+/*
+ * Check if an event is set at the phase on the server. If there is, set
+ * elevel and return true.
+ */
+static bool
+check_event(char *servername, char *phase, int *elevel)
+{
+	LWLockAcquire(fxss->lock, LW_SHARED);
+
+	if (pg_strcasecmp(fxss->server, servername) != 0 ||
+		pg_strcasecmp(fxss->phase, phase) != 0)
+	{
+		LWLockRelease(fxss->lock);
+		return false;
+	}
+
+	/* Currently support only error and panic */
+	if (strcasecmp(fxss->elevel, "error") == 0)
+		*elevel = ERROR;
+	if (strcasecmp(fxss->elevel, "panic") == 0)
+		*elevel = PANIC;
+
+	LWLockRelease(fxss->lock);
+
+	return true;
+}
+
+/* SQL function to inject an error */
+Datum
+test_inject_error(PG_FUNCTION_ARGS)
+{
+	char *elevel = text_to_cstring(PG_GETARG_TEXT_P(0));
+	char *phase = text_to_cstring(PG_GETARG_TEXT_P(1));
+	char *server = text_to_cstring(PG_GETARG_TEXT_P(2));
+
+	LWLockAcquire(fxss->lock, LW_EXCLUSIVE);
+	strncpy(fxss->elevel, elevel, TEST_FDWXCT_MAX_NAME_LEN);
+	strncpy(fxss->phase, phase, TEST_FDWXCT_MAX_NAME_LEN);
+	strncpy(fxss->server, server, TEST_FDWXCT_MAX_NAME_LEN);
+	LWLockRelease(fxss->lock);
+
+	PG_RETURN_NULL();
+}
+
+/* SQL function to reset an error */
+Datum
+test_reset_error(PG_FUNCTION_ARGS)
+{
+	LWLockAcquire(fxss->lock, LW_EXCLUSIVE);
+	memset(fxss->elevel, 0, TEST_FDWXCT_MAX_NAME_LEN);
+	memset(fxss->phase, 0, TEST_FDWXCT_MAX_NAME_LEN);
+	memset(fxss->server, 0, TEST_FDWXCT_MAX_NAME_LEN);
+	LWLockRelease(fxss->lock);
+
+	PG_RETURN_NULL();
+}
diff --git a/src/test/modules/test_fdwxact/test_fdwxact.control b/src/test/modules/test_fdwxact/test_fdwxact.control
new file mode 100644
index 0000000000..ac9945ba03
--- /dev/null
+++ b/src/test/modules/test_fdwxact/test_fdwxact.control
@@ -0,0 +1,4 @@
+comment = 'Test code for fdwxact'
+default_version = '1.0'
+module_pathname = '$libdir/test_fdwxact'
+relocatable = true
diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index fa8e031526..d47d96975b 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/021_fdwxact.pl b/src/test/recovery/t/021_fdwxact.pl
new file mode 100644
index 0000000000..9af9bb81dc
--- /dev/null
+++ b/src/test/recovery/t/021_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 38b2b1e8e1..f30fe6b492 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2335,9 +2335,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2352,7 +2355,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m %b[%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.23.0

From c7460c761ae23245cc500f8ada5964b7d13aeec7 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 21:28:58 +0500
Subject: [PATCH v21 4/5] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/Makefile                 |   7 +-
 contrib/postgres_fdw/connection.c             | 603 +++++++++++-------
 .../postgres_fdw/expected/postgres_fdw.out    | 280 +++++++-
 contrib/postgres_fdw/fdwxact.conf             |   3 +
 contrib/postgres_fdw/postgres_fdw.c           |  21 +-
 contrib/postgres_fdw/postgres_fdw.h           |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql     | 124 +++-
 doc/src/sgml/postgres-fdw.sgml                |  45 ++
 8 files changed, 831 insertions(+), 259 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index ee8a80a392..91fa6e39fc 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -16,7 +16,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -29,3 +29,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index 52d1fe3563..d55884b49b 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2020, PostgreSQL Global Development Group
  *
@@ -12,6 +12,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "catalog/pg_user_mapping.h"
@@ -56,6 +57,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -69,17 +71,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 								   SubTransactionId mySubid,
 								   SubTransactionId parentSubid,
@@ -92,23 +90,26 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 									 PGresult **result);
 static bool UserMappingPasswordRequired(UserMapping *user);
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +129,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,12 +136,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -155,6 +149,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -182,6 +191,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +200,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +211,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,11 +227,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -473,7 +521,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -700,193 +748,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 	PG_END_TRY();
 }
 
-/*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow any remote transactions, since it's not
-					 * very reasonable to hold them open until the prepared
-					 * transaction is committed.  For the moment, throw error
-					 * unconditionally; later we might allow read-only cases.
-					 * Note that the error will cause us to come right back
-					 * here with event == XACT_EVENT_ABORT, so we'll clean up
-					 * the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot PREPARE a transaction that has operated on postgres_fdw foreign tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
 /*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
@@ -903,10 +764,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -917,6 +774,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1251,3 +1112,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recursively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 90db550b92..8c31e26406 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -191,15 +210,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8923,10 +8944,10 @@ RESET ROLE;
 ALTER USER MAPPING FOR regress_nosuper SERVER loopback_nopw OPTIONS (ADD password_required 'false');
 SET ROLE regress_nosuper;
 -- Should finally work now
-SELECT * FROM ft1_nopw LIMIT 1;
-  c1  | c2 | c3 | c4 | c5 | c6 |     c7     | c8 
-------+----+----+----+----+----+------------+----
- 1111 |  2 |    |    |    |    | ft1        | 
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
+ c1 | c2 |        c3         |              c4              |            c5            | c6 |     c7     | c8  
+----+----+-------------------+------------------------------+--------------------------+----+------------+-----
+  1 |  2 | 00001_trig_update | Fri Jan 02 00:00:00 1970 PST | Fri Jan 02 00:00:00 1970 | 1  | 1          | foo
 (1 row)
 
 -- unpriv user also cannot set sslcert / sslkey on the user mapping
@@ -8943,16 +8964,16 @@ HINT:  User mappings with the sslcert or sslkey options set may only be created
 DROP USER MAPPING FOR CURRENT_USER SERVER loopback_nopw;
 -- This will fail again as it'll resolve the user mapping for public, which
 -- lacks password_required=false
-SELECT * FROM ft1_nopw LIMIT 1;
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
 ERROR:  password is required
 DETAIL:  Non-superusers must provide a password in the user mapping.
 RESET ROLE;
 -- The user mapping for public is passwordless and lacks the password_required=false
 -- mapping option, but will work because the current user is a superuser.
 SELECT * FROM ft1_nopw LIMIT 1;
-  c1  | c2 | c3 | c4 | c5 | c6 |     c7     | c8 
-------+----+----+----+----+----+------------+----
- 1111 |  2 |    |    |    |    | ft1        | 
+ c1 | c2 |  c3   |              c4              |            c5            | c6 |     c7     | c8  
+----+----+-------+------------------------------+--------------------------+----+------------+-----
+  6 |  6 | 00006 | Wed Jan 07 00:00:00 1970 PST | Wed Jan 07 00:00:00 1970 | 6  | 6          | foo
 (1 row)
 
 -- cleanup
@@ -8961,16 +8982,225 @@ DROP OWNED BY regress_nosuper;
 DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
--- Two-phase transactions are not supported.
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
 BEGIN;
-SELECT count(*) FROM ft1;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" of relation "T 5" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
  count 
 -------
-   822
+     0
 (1 row)
 
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
-ERROR:  cannot PREPARE a transaction that has operated on postgres_fdw foreign tables
-ROLLBACK;
-WARNING:  there is no transaction in progress
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000000..3fdbf93cdb
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 9fc53cad68..105451d199 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include <limits.h>
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -504,7 +505,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 							  const PgFdwRelationInfo *fpinfo_o,
 							  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -558,6 +558,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1434,7 +1439,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2372,7 +2377,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2746,7 +2751,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3566,7 +3571,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4441,7 +4446,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4527,7 +4532,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4755,7 +4760,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index eef410db39..43ffd4f73f 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "libpq-fe.h"
@@ -129,7 +130,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -137,6 +138,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 							   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int	ExtractConnectionOptions(List *defelems,
@@ -203,6 +207,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 									bool is_subquery,
 									List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 83971665e3..1ef66123df 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2598,7 +2621,7 @@ ALTER USER MAPPING FOR regress_nosuper SERVER loopback_nopw OPTIONS (ADD passwor
 SET ROLE regress_nosuper;
 
 -- Should finally work now
-SELECT * FROM ft1_nopw LIMIT 1;
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
 
 -- unpriv user also cannot set sslcert / sslkey on the user mapping
 -- first set password_required so we see the right error messages
@@ -2612,7 +2635,7 @@ DROP USER MAPPING FOR CURRENT_USER SERVER loopback_nopw;
 
 -- This will fail again as it'll resolve the user mapping for public, which
 -- lacks password_required=false
-SELECT * FROM ft1_nopw LIMIT 1;
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
 
 RESET ROLE;
 
@@ -2628,9 +2651,98 @@ DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
 
--- Two-phase transactions are not supported.
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
 BEGIN;
-SELECT count(*) FROM ft1;
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
 ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index eab2cc9378..3ea3ce9335 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -477,6 +477,43 @@ OPTIONS (ADD password_required 'false');
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -504,6 +541,14 @@ OPTIONS (ADD password_required 'false');
    managed by creating corresponding remote savepoints.
   </para>
 
+  <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
   <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
-- 
2.23.0

From 111984cb33ebeec8a8f9a01d73c7d0b935429e42 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Tue, 19 May 2020 15:16:02 +0900
Subject: [PATCH v21 2/5] Support atomic commit among multiple foreign servers.

---
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  110 +
 src/backend/access/fdwxact/fdwxact.c          | 2771 +++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  566 ++++
 src/backend/access/fdwxact/resolver.c         |  436 +++
 src/backend/access/rmgrdesc/Makefile          |    1 +
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   58 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   56 +
 src/backend/access/transam/xact.c             |   29 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |    3 +
 src/backend/commands/copy.c                   |    6 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    8 +
 src/backend/executor/nodeForeignscan.c        |   24 +
 src/backend/executor/nodeModifyTable.c        |    6 +
 src/backend/foreign/foreign.c                 |   55 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   18 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   80 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/fdwxactdesc.c              |    1 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  163 +
 src/include/access/fdwxact_launcher.h         |   28 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   54 +
 src/include/access/resolver_internal.h        |   63 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   22 +
 src/include/foreign/fdwapi.h                  |   12 +
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    6 +
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    2 +
 src/test/regress/expected/rules.out           |    7 +
 55 files changed, 4834 insertions(+), 17 deletions(-)
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100644 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 120000 src/bin/pg_waldump/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a8bb..49480dd039 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000000..0207a66fb4
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000000..d9f08f4cfa
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,110 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a consistent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+---------------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consists
+of the following four steps:
+
+1. Foreign Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant. The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+We record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE each foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared transaction on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step (commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itself to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rollbacked the resolver process wake up the waiter.
+
+
+Foreign Data Wrapper Callbacks for Transaction Management
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForeignTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for PREPARE, COMMIT or ROLLBACK
+the transaction on the foreign server, respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transaction in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions will have an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_PREPARING
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING before the foreign
+transaction is committed and aborted by FDW callback functions respectively.
+FdwXact entry is removed once the foreign transaction is resolved with WAL
+logging.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*1). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                     PREPARING                      |----+
+ +----------------------------------------------------+    |
+                          |                                |
+                          v                                |
+ +----------------------------------------------------+    |
+ |                    PREPARED(*1)                    |    | (*2)
+ +----------------------------------------------------+    |
+           |                               |               |
+           v                               v               |
+ +--------------------+          +--------------------+    |
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |<---+
+ +--------------------+          +--------------------+
+
+(*1) Recovered FdwXact entries starts with PREPARED
+(*2) Paths when an error occurrs during preparing
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100644
index 0000000000..698bd9cff4
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2771 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers atomically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * Two-phase commit protocol is used when the transaction modified two or
+ * more servers including the local node.  If two-phase commit protocol
+ * is not required all foreign transactions are committed at pre-commit
+ * phase.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit.  The foreign servers are
+ * registered if FDW has both CommitForeignTransaction API and
+ * RollbackForeignTransaction API.  Registered participant servers are
+ * identified by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * all foreign servers.  And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask to commit, we also tell to notify us when
+ * it's done, so that we can wait interruptibly to finish, and so that
+ * we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request.  We have one queue
+ * of which elements are ordered by the timestamp when they expect to be
+ * processed.  Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp when they expects to be processed.
+ * On failure, it enqueues again with new timestamp (last timestamp +
+ * foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction).  Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by
+ * pg_resovle_foreign_xact() function.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed, the corresponding FdwXact
+ * entry is update.  To avoid holding the lock during transaction processing
+ * which may take an unpredicatable time the in-memory data of foreign
+ * transaction follows a locking model based on four linked concepts:
+ *
+ * * All FdwXact fields except for indoubt, inprocessing and status are protected
+ *   by FdwXactLock.  These three fields are protected by its mutex.
+ * * Setting held_by of an FdwXact entry means to own the FdwXact entry, which
+ *   prevent it from updated and removed by concurrent processes.
+ * * The FdwXact whose inprocessing is true is also not processed or removed
+ *   by concurrent processes.
+ * * A process who is going to process foreign transaction needs to hold its
+ *   FdwXact entry in advance.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk.  FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon.  We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts.  If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to disk.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED)
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define ServerSupportTransactionCallack(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->commit_foreign_xact_fn != NULL)
+#define SeverSupportTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/*
+ * Structure to bundle the foreign transaction participant.  This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry is
+	 * not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer *server;
+	UserMapping *usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char	   *fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool		modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function prepare_foreign_xact_fn;
+	CommitForeignTransaction_function commit_foreign_xact_fn;
+	RollbackForeignTransaction_function rollback_foreign_xact_fn;
+	GetPrepareId_function get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transactions involved in the transaction.  A member of
+ * participants may not support transaction callbacks: commit, rollback and
+ * prepare.  If a member of participants doesn't support any transaction
+ * callbacks, i.g. ServerSupportTransactionCallack() returns false,
+ * we don't end its transaction.
+ *
+ * FdwXactParticipants_tmp is used to update FdwXactParticipants atomically
+ * when executing COMMIT/ROLLBACK PREPARED command.  In COMMIT PREPARED case,
+ * we don't want to rollback foreign transactions even if an error occurs,
+ * because the local prepared transaction never turn over rollback in that
+ * case.  However, preparing FdwXactParticipants might be lead an error
+ * because of calling palloc() inside.  So we prepare FdwXactParticipants in
+ * two phase.  In the first phase, PrepareFdwXactParticipants(), we collect
+ * all foreign transactions associated with the local prepared transactions
+ * and kept them in FdwXactParticipants_tmp.  Even if an error occurs during
+ * that, we don't rollback them.  In the second phase, SetFdwXactParticipants(),
+ * we replace FdwXactParticipants_tmp with FdwXactParticipants and hold them.
+ *
+ * FdwXactLocalXid is the local transaction id associated with FdwXactParticipants.
+ */
+static List *FdwXactParticipants = NIL;
+static List *FdwXactParticipants_tmp = NIL;
+static TransactionId FdwXactLocalXid = InvalidTransactionId;
+
+/*
+ * True is the current transaction needs to be committed together with
+ * foreign servers.
+ */
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int			max_prepared_foreign_xacts = 0;
+int			max_foreign_xact_resolvers = 0;
+int			foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static void FdwXactParticipantEndTransaction(FdwXactParticipant *fdw_part, bool onephase,
+											 bool for_commit);
+static bool checkForeignTwophaseCommitRequired(void);
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static void FdwXactPrepareForeignTransactions(bool prepare_all);
+static void FdwXactResolveOneFdwXact(FdwXact fdwxact);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(bool mark_indoubt);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid, void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static TransactionId FdwXactDetermineTransactionFate(TransactionId xid);
+static int	get_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						bool hold);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXactParticipant *create_fdwxact_participant(Oid serverid, Oid userid,
+													  FdwRoutine *routine);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&fdwxacts[cnt].mutex);
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation	rel;
+	Oid			serverid;
+	Oid			userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction. The foreign transaction identified
+ * by given server id and user id.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant *fdw_part;
+	MemoryContext old_ctx;
+	FdwRoutine *routine;
+	ListCell   *lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	fdw_part = create_fdwxact_participant(serverid, userid, routine);
+	fdw_part->modified = modified;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	pfree(routine);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/* Return palloc'd FdwXactParticipant variable */
+static FdwXactParticipant *
+create_fdwxact_participant(Oid serverid, Oid userid, FdwRoutine *routine)
+{
+	FdwXactParticipant *fdw_part;
+	ForeignServer *foreign_server;
+	UserMapping *user_mapping;
+
+	foreign_server = GetForeignServer(serverid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->modified = false;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	return fdw_part;
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * When foreign twophase commit is enabled, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign servers' FDW to support two-phase commit protocol and ask them to
+ * prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and when 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXact(void)
+{
+	ListCell   *lc;
+	bool		need_twophase_commit;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false if
+	 * foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = checkForeignTwophaseCommitRequired();
+
+	/*
+	 * Prepare foreign transactions on foreign servers that support two-phase
+	 * commit.
+	 */
+	if (need_twophase_commit)
+	{
+		FdwXactPrepareForeignTransactions(false);
+		ForeignTwophaseCommitIsRequired = true;
+	}
+
+	/*
+	 * Commit other foreign transactions and delete the participant entry from
+	 * the list.
+	 */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		/*
+		 * Skip already prepared foreign transactions. Note that we keep those
+		 * FdwXactParticipants until the end of the transaction.
+		 */
+		if (fdw_part->fdwxact)
+			continue;
+
+		/* Delete non-transaction-support participants */
+		if (!ServerSupportTransactionCallack(fdw_part))
+		{
+			FdwXactParticipants = foreach_delete_current(FdwXactParticipants, lc);
+			continue;
+		}
+
+		/* Commit the foreign transaction in one-phase */
+		FdwXactParticipantEndTransaction(fdw_part, true, true);
+
+		/* Transaction successfully committed delete from the participant list */
+		FdwXactParticipants = foreach_delete_current(FdwXactParticipants, lc);
+	}
+}
+
+/*
+ * Return true if the current transaction modifies data on two or more servers
+ * in FdwXactParticipants and local server itself.
+ */
+static bool
+checkForeignTwophaseCommitRequired(void)
+{
+	ListCell   *lc;
+	bool		need_twophase_commit;
+	bool		have_notwophase;
+	int			nserverswritten = 0;
+	int			nserverstwophase = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (!fdw_part->modified)
+			continue;
+
+		if (SeverSupportTwophaseCommit(fdw_part))
+			nserverstwophase++;
+
+		nserverswritten++;
+	}
+	Assert(nserverswritten >= nserverstwophase);
+
+	/* check if there is any servers that don't support two-phase commit */
+	have_notwophase = (nserverswritten != nserverstwophase);
+
+	/* Did we modify the local non-temporary data? */
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+	{
+		nserverswritten++;
+
+		/*
+		 * We increment nserverstwophase as well for making code simple,
+		 * although we don't actually use two-phase commit for the local
+		 * transaction.
+		 */
+		nserverstwophase++;
+	}
+
+	if (nserverswritten <= 1)
+		return false;
+
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED)
+	{
+		/*
+		 * In 'required' case, we require for all modified server to support
+		 * two-phase commit.
+		 */
+		need_twophase_commit = (nserverswritten >= 2);
+	}
+	else
+	{
+		Assert(foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER);
+
+		/*
+		 * In 'prefer' case, we use two-phase commit when this transaction modified
+		 * two or more servers including the local server or servers that support
+		 * two-phase commit.
+		 */
+		need_twophase_commit = (nserverstwophase >= 2);
+	}
+
+	/*
+	 * If foreign two phase commit is required then all foreign serves must be
+	 * capable of doing two-phase commit
+	 */
+	if (need_twophase_commit)
+	{
+		/* Parameter check */
+		if (max_prepared_foreign_xacts == 0)
+			ereport(ERROR,
+					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					 errmsg("foreign two-phase commit is required but prepared foreign transactions are disabled"),
+					 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+		if (max_foreign_xact_resolvers == 0)
+			ereport(ERROR,
+					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					 errmsg("foreign two-phase commit is required but prepared foreign transactions are disabled"),
+					 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+		if (have_notwophase &&
+			foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					 errmsg("cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol"),
+					 errdetail("foreign_twophase_commit is \'required\' but the transaction has some foreign servers which are not capable of two-phase commit")));
+	}
+
+	return need_twophase_commit;
+}
+
+/*
+ * The routine for committing or rolling back the given transaction participant.
+ */
+static void
+FdwXactParticipantEndTransaction(FdwXactParticipant *fdw_part, bool onephase,
+								 bool for_commit)
+{
+	FdwXactRslvState state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state.xid = FdwXactLocalXid;
+	state.server = fdw_part->server;
+	state.usermapping = fdw_part->usermapping;
+	state.fdwxact_id = onephase ? NULL : fdw_part->fdwxact_id;
+	state.flags = onephase ? FDWXACT_FLAG_ONEPHASE : 0;
+
+	if (for_commit)
+	{
+		fdw_part->commit_foreign_xact_fn(&state);
+		elog(DEBUG1, "successfully committed the foreign transaction for server %u user %u",
+			 fdw_part->usermapping->serverid,
+			 fdw_part->usermapping->userid);
+	}
+	else
+	{
+		fdw_part->rollback_foreign_xact_fn(&state);
+		elog(DEBUG1, "successfully rolled back the foreign transaction for server %u user %u",
+			 fdw_part->usermapping->serverid,
+			 fdw_part->usermapping->userid);
+	}
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here on failure. If any error occurs, we
+ * rollback non-prepared foreign transactions.
+ */
+static void
+FdwXactPrepareForeignTransactions(bool prepare_all)
+{
+	ListCell   *lc;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Save the local transaction id */
+	FdwXactLocalXid = GetTopTransactionId();
+
+	/* Loop over the foreign connections */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		FdwXactRslvState state;
+		FdwXact		fdwxact;
+
+		if (!SeverSupportTwophaseCommit(fdw_part))
+			continue;
+
+		if (!prepare_all && !fdw_part->modified)
+			continue;
+
+		/* Get prepared transaction identifier */
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, FdwXactLocalXid);
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the
+		 * FDWXACT_STATUS_PREPARING status. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before
+		 * persisting the information to the disk and crash in-between these
+		 * two steps, we will lost the prepared transaction on the foreign
+		 * server and will not be able to resolve it after the crash recovery.
+		 * Hence persist first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(FdwXactLocalXid, fdw_part);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 */
+		state.xid = FdwXactLocalXid;
+		state.server = fdw_part->server;
+		state.usermapping = fdw_part->usermapping;
+		state.fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+		fdw_part->prepare_foreign_xact_fn(&state);
+
+		/* succeeded, update status */
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		SpinLockRelease(&fdwxact->mutex);
+	}
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact		fdwxact;
+	FdwXactOnDiskData *fdwxact_file_data;
+	MemoryContext old_context;
+	int			data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							 fdw_part->usermapping->userid,
+							 fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_PREPARING;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyProc->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyProc->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			   Oid umid, char *fdwxact_id)
+{
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->valid &&
+			fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("Duplicate entry with transaction id %u, serverid %u, userid %u exists.",
+									  xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->local_xid = xid;
+	fdwxact->dbid = dbid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->inprocessing = false;
+	fdwxact->indoubt = false;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int			i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("Failed to find entry for xid %u, foreign server %u, and user %u.",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the WAL
+		 * record is inserted could complete without fsync'ing our state file.
+		 * (This is essentially the same kind of race condition as the
+		 * COMMIT-to-clog-write case that RecordTransactionCommit uses
+		 * delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyProc->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and remove
+		 * the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/* Now we can mark ourselves as out of the commit critical section */
+		MyProc->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	   *id;
+	int			id_len = 0;
+
+	/*
+	 * If FDW doesn't provide the callback function, generate an unique
+	 * identifier.
+	 */
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char		buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+				 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifier \"%s\" is too long",
+						id),
+				 errdetail("Foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * Prepare foreign transactions by PREPARE TRANSACTION command.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXact(void)
+{
+	ListCell   *lc;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	/*
+	 * Check if there is a server that doesn't support two-phase commit. All involved
+	 * servers need to support two-phase commit as we prepare on them regardless of
+	 * modified or not.
+	 */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (!SeverSupportTwophaseCommit(fdw_part))
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					 errmsg("cannot PREPARE a distributed transaction that has operated on a foreign server")));
+	}
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions(true);
+
+	/*
+	 * We keep prepared foreign transaction participants to rollback them in case
+	 * of failure.
+	 */
+}
+
+void
+PostPrepare_FdwXact(void)
+{
+	/* After preparing the local transaction, we can forget all participants */
+	ForgetAllFdwXactParticipants(false);
+}
+
+/*
+ * Collect all foreign transactions associated with the given xid.  Return true
+ * if COMMIT PREPARED or ROLLBACK PREPARED needs to wait for all foreign transactions
+ * to be resolved.  The collected foreign transactions are kept in FdwXactParticipants_tmp,
+ * so the caller must call SetFdwXactParticipants() later if this function returns true.
+ */
+bool
+PrepareFdwXactParticipants(TransactionId xid)
+{
+	MemoryContext old_ctx;
+
+	Assert(FdwXactParticipants_tmp == NIL);
+
+	if (!TwoPhaseExists(xid))
+		return false;
+
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXactParticipant *fdw_part;
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		FdwRoutine *routine;
+
+		if (!fdwxact->valid || fdwxact->local_xid != xid)
+			continue;
+
+		routine = GetFdwRoutineByServerId(fdwxact->serverid);
+		fdw_part = create_fdwxact_participant(fdwxact->serverid, fdwxact->userid,
+											  routine);
+		fdw_part->modified = true;
+		fdw_part->fdwxact = fdwxact;
+
+		/* Add to the participants list */
+		FdwXactParticipants_tmp = lappend(FdwXactParticipants_tmp, fdw_part);
+	}
+	LWLockRelease(FdwXactLock);
+
+	MemoryContextSwitchTo(old_ctx);
+
+	/*
+	 * We cannot proceed to commit this prepared transaction when
+	 * foreign_twophase_commit is disabled.
+	 */
+	if (FdwXactParticipants_tmp != NIL &&
+		!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot process a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	return (FdwXactParticipants_tmp != NIL);
+}
+
+/*
+ * Make the collected foreign transactions the participants of this transaction and
+ * hold all of them.  This function must be called after PrepareFdwXactParticipants().
+ */
+void
+SetFdwXactParticipants(TransactionId xid, bool commit)
+{
+	ListCell   *lc;
+
+	Assert(FdwXactParticipants_tmp != NIL);
+	Assert(FdwXactParticipants == NIL);
+
+	FdwXactLocalXid = xid;
+	FdwXactParticipants = FdwXactParticipants_tmp;
+	FdwXactParticipants_tmp = NIL;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		Assert(SeverSupportTwophaseCommit(fdw_part));
+
+		/* Hold the fdwxact entry and set the status */
+		SpinLockAcquire(&fdw_part->fdwxact->mutex);
+		Assert(fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED);
+		fdw_part->fdwxact->held_by = MyBackendId;
+		fdw_part->fdwxact->status = commit
+			? FDWXACT_STATUS_COMMITTING
+			: FDWXACT_STATUS_ABORTING;
+		SpinLockRelease(&fdw_part->fdwxact->mutex);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants(true);
+}
+
+/*
+ * Wait for its all foreign transactions to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitForResolution(TransactionId wait_xid)
+{
+	char	   *new_status = NULL;
+	const char *old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/*
+	 * Quick exit if either atomic commit is not requested or we don't have
+	 * any participants.
+	 */
+	if (!IsForeignTwophaseCommitRequested() || FdwXactParticipants == NIL)
+		return;
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction resolution.
+	 */
+	if (update_process_title)
+	{
+		int			len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status);
+		new_status[len] = '\0'; /* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The
+		 * latter would lead the client to believe that the distributed
+		 * transaction aborted, which is not true: it's already committed
+		 * locally. The former is no good either: the client has requested
+		 * committing a distributed transaction, and is entitled to assume
+		 * that a acknowledged commit is also commit on all foreign servers,
+		 * which might not be true. So in this case we issue a WARNING (which
+		 * some clients may be able to interpret) and shut off further output.
+		 * We do NOT reset PorcDiePending, so that the process will die after
+		 * the commit is cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve
+		 * them later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status);
+		pfree(new_status);
+	}
+
+	list_free(FdwXactParticipants);
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz now, TimestampTz *nextResolutionTs_p,
+				 TransactionId *waitXid_p)
+{
+	PGPROC	   *proc;
+	bool		found = false;
+
+	Assert(LWLockHeldByMe(FdwXactResolutionLock));
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	/* Initialize variables */
+	*nextResolutionTs_p = -1;
+	*waitXid_p = InvalidTransactionId;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+		{
+			if (proc->fdwXactNextResolutionTs <= now)
+			{
+				/* Found a waiting process */
+				found = true;
+				*waitXid_p = proc->fdwXactWaitXid;
+			}
+			else
+				/* Found a waiting process supposed to be processed later */
+				*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+
+			break;
+		}
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return found ? proc : NULL;
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC	   *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC	   *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC	   *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * In abort case, this function ends foreign transaction participants and possibly
+ * rollback their prepared foreign trasnactions.
+ */
+extern void
+AtEOXact_FdwXact(bool is_commit)
+{
+	ListCell   *lc;
+
+	if (!is_commit)
+	{
+		foreach(lc, FdwXactParticipants)
+		{
+			FdwXactParticipant *fdw_part = lfirst(lc);
+			FdwXact		fdwxact = fdw_part->fdwxact;
+			int			status;
+
+			if (!fdwxact)
+			{
+				/* Rollback foreign transaction in one-phase if supported */
+				if (ServerSupportTransactionCallack(fdw_part))
+					FdwXactParticipantEndTransaction(fdw_part, true, false);
+				continue;
+			}
+
+			/*
+			 * Abort the foreign transaction.  For participants whose status
+			 * is FDWXACT_STATUS_PREPARING, we close the transaction in
+			 * one-phase. In addition, since we are not sure that the
+			 * preparation has been completed on the foreign server, we also
+			 * attempts to rollback the prepared foreign transaction.  Note
+			 * that it's FDWs responsibility that they tolerate
+			 * OBJECT_NOT_FOUND error in abort case.
+			 */
+			SpinLockAcquire(&fdwxact->mutex);
+			status = fdwxact->status;
+			fdwxact->status = FDWXACT_STATUS_ABORTING;
+			SpinLockRelease(&fdwxact->mutex);
+
+			switch (status)
+			{
+				case FDWXACT_STATUS_PREPARING:
+					/* One-phase rollback foreign transaction */
+					FdwXactParticipantEndTransaction(fdw_part, true, false);
+					/* FALLTHROUGH */
+				case FDWXACT_STATUS_PREPARED:
+				case FDWXACT_STATUS_ABORTING:
+					/* One-phase rollback foreign transaction */
+					FdwXactParticipantEndTransaction(fdw_part, false, false);
+					break;
+				case FDWXACT_STATUS_COMMITTING:
+					Assert(false);
+					break;
+			}
+
+			/* Resolution was a success, remove the entry */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			if (fdwxact->ondisk)
+				RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								  fdwxact->serverid, fdwxact->userid,
+								  true);
+			remove_fdwxact(fdwxact);
+			LWLockRelease(FdwXactLock);
+		}
+
+		/* All foreign transaction should be aborted */
+		list_free(FdwXactParticipants);
+		FdwXactParticipants = NIL;
+	}
+
+	ForgetAllFdwXactParticipants(true);
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(bool mark_indoubt)
+{
+	ListCell   *cell;
+	int			nlefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact		fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		Assert(fdwxact);
+
+		/*
+		 * Unlock and mark a foreign transaction as in-doubt.  Note that there
+		 * is a race condition; the FdwXact entries in FdwXactParticipants
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget. So we need to check if the entries are still
+		 * associated with the transaction.  Also we do these check by
+		 * transaction id because these foreign transaction may already be
+		 * held by the resolver.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		if (fdwxact->valid && fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+
+			if (mark_indoubt)
+			{
+				fdwxact->indoubt = true;	/* let resolver to process */
+				nlefts++;
+			}
+		}
+		LWLockRelease(FdwXactLock);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (nlefts > 0)
+	{
+		Assert(mark_indoubt);
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", nlefts);
+		FdwXactComputeRequiredXmin();
+		FdwXactLaunchOrWakeupResolver();
+	}
+
+	list_free(FdwXactParticipants);
+	FdwXactParticipants = NIL;
+	FdwXactParticipants_tmp = NIL;
+	FdwXactLocalXid = InvalidTransactionId;
+}
+
+/*
+ * Resolve foreign transactions at the give indexes. If 'waiter' is not NULL,
+ * we release the waiter after we resolved all of the given foreign transactions
+ * On failure we re-enqueue the waiting backend after incremented the next
+ * resolution time.
+ *
+ * The caller must hold the given foreign transactions in advance to prevent
+ * concurrent update.
+ */
+void
+FdwXactResolveFdwXacts(int *fdwxact_idxs, int nfdwxacts, PGPROC *waiter)
+{
+	for (int i = 0; i < nfdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[fdwxact_idxs[i]];
+
+		CHECK_FOR_INTERRUPTS();
+
+		PG_TRY();
+		{
+			FdwXactResolveOneFdwXact(fdwxact);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			if (waiter)
+			{
+				LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+				if (waiter->fdwXactState == FDWXACT_WAITING)
+				{
+					SHMQueueDelete(&(waiter->fdwXactLinks));
+					pg_write_barrier();
+					waiter->fdwXactNextResolutionTs =
+						TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+													foreign_xact_resolution_retry_interval);
+					FdwXactQueueInsert(waiter);
+				}
+				LWLockRelease(FdwXactResolutionLock);
+			}
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		if (fdwxact->ondisk)
+			RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+							  fdwxact->userid, true);
+		remove_fdwxact(fdwxact);
+		LWLockRelease(FdwXactLock);
+	}
+
+	if (!waiter)
+		return;
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter could
+	 * already be detached if user cancelled to wait before resolution.
+	 */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/*
+		 * Wake up the waiter only when we have set state and removed from
+		 * queue
+		 */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+FdwXactExists(Oid dbid, Oid serverid, Oid userid)
+{
+	int			idx;
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	idx = get_fdwxact(dbid, InvalidTransactionId, serverid, userid,
+					  false);
+	LWLockRelease(FdwXactLock);
+
+	return (idx != -1);
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ *
+ * XXX: we can exclude FdwXact entries whose status is already committing
+ * or aborting.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+
+/*
+ * Determine whether the foreign transaction associated with the given transaction
+ * id should be committed or rolled back according to the result of the local
+ * transaction.
+ */
+static FdwXactStatus
+FdwXactDetermineTransactionFate(TransactionId xid)
+{
+	/*
+	 * If the local transaction is already committed, commit prepared foreign
+	 * transaction.
+	 */
+	if (TransactionIdDidCommit(xid))
+		return FDWXACT_STATUS_COMMITTING;
+
+	/*
+	 * If the local transaction is already aborted, abort prepared foreign
+	 * transactions.
+	 */
+	else if (TransactionIdDidAbort(xid))
+		return FDWXACT_STATUS_ABORTING;
+
+
+	/*
+	 * The local transaction is not in progress but the foreign transaction is
+	 * not prepared on the foreign server. This can happen when transaction
+	 * failed after registered this entry but before actual preparing on the
+	 * foreign server. So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(xid))
+		return FDWXACT_STATUS_ABORTING;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is about
+	 * to be committed or aborted. This should not happen except for one case
+	 * where the local transaction is prepared and this foreign transaction is
+	 * being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		elog(ERROR,
+			 "cannot resolve the foreign transaction associated with in-process transaction");
+
+	pg_unreachable();
+}
+
+/*
+ * Commit or rollback one prepared foreign transaction.  After resolved
+ * successfully, the FdwXact entry is removed from the shared memory and also
+ * remove the corresponding on-disk file.
+ */
+static void
+FdwXactResolveOneFdwXact(FdwXact fdwxact)
+{
+	FdwXactRslvState state;
+	ForeignServer *server;
+	ForeignDataWrapper *fdw;
+	FdwRoutine *routine;
+
+	Assert(fdwxact != NULL);
+	Assert(fdwxact->held_by != InvalidBackendId || fdwxact->inprocessing);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+	{
+		FdwXactStatus new_status;
+
+		new_status = FdwXactDetermineTransactionFate(fdwxact->local_xid);
+		Assert(new_status == FDWXACT_STATUS_COMMITTING ||
+			   new_status == FDWXACT_STATUS_ABORTING);
+
+		/* Update the status */
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwxact->status = new_status;
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	routine = GetFdwRoutine(fdw->fdwhandler);
+
+	/* Prepare resolution state to pass to API */
+	state.xid = fdwxact->local_xid;
+	state.server = server;
+	state.usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+	state.fdwxact_id = fdwxact->fdwxact_id;
+	state.flags = 0;
+
+	if (fdwxact->status == FDWXACT_STATUS_COMMITTING)
+	{
+		routine->CommitForeignTransaction(&state);
+		elog(DEBUG1, "successfully committed the prepared foreign transaction for server %u user %u",
+			 fdwxact->serverid, fdwxact->userid);
+	}
+	else
+	{
+		routine->RollbackForeignTransaction(&state);
+		elog(DEBUG1, "successfully rolled back the prepared foreign transaction for server %u user %u",
+			 fdwxact->serverid, fdwxact->userid);
+	}
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respectively.
+ */
+static int
+get_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid, bool hold)
+{
+	bool		found = false;
+	int			i;
+
+	Assert(LWLockHeldByMe(FdwXactLock));
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		bool		inprocessing;
+
+		if (!fdwxact->valid)
+			continue;
+
+		SpinLockAcquire(&fdwxact->mutex);
+		inprocessing = fdwxact->inprocessing;
+		SpinLockRelease(&fdwxact->mutex);
+
+		/*
+		 * If we're attempting to hold this entry, skip if it is already held
+		 * or being processed.
+		 */
+		if (hold &&
+			(inprocessing || fdwxact->held_by != InvalidBackendId))
+			continue;
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* This entry matches the condition */
+
+		if (hold)
+			fdwxact->held_by = MyBackendId;
+
+		found = true;
+		break;
+	}
+
+	return found ? i : -1;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record in
+		 * FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transaction log record", info);
+
+	return;
+}
+
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The status of
+	 * the transaction is set as preparing, since we do not know the exact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction which prepared this foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							 fdwxact_data->serverid, fdwxact_data->userid,
+							 fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know the xact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;		/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact		fdwxact;
+	int			i;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->dbid == dbid && fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid && fdwxact->userid == userid)
+			break;
+	}
+
+	if (i >= FdwXactCtl->num_fdwxacts)
+		return;
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint's redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;					/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence of long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked invalid,
+	 * because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+				(errmsg_plural("%u foreign transaction state file was written "
+							   "for long-running prepared transactions",
+							   "%u foreign transaction state files were written "
+							   "for long-running prepared transactions",
+							   serialized_fdwxacts,
+							   serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
+									XL_ROUTINE(.page_read = &read_local_xlog_page,
+											   .segment_open = &wal_segment_open,
+											   .segment_close = &wal_segment_close),
+									NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+				 errdetail("Failed while allocating an XLog reading processor.")));
+
+	XLogBeginRead(xlogreader, lsn);
+	record = XLogReadRecord(xlogreader, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not read foreign transaction state from xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not recreate foreign transaction state file \"%s\": %m",
+						path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId origNextXid =
+	XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	   *buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open FDW transaction state file \"%s\": %m",
+						path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char	   *buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign prepared transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+}			WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	6
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		FdwXactStatus status;
+		bool		indoubt;
+		char	   *xact_status;
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+
+		if (!fdwxact->valid)
+			continue;
+
+		memset(nulls, 0, sizeof(nulls));
+
+		SpinLockAcquire(&fdwxact->mutex);
+		status = fdwxact->status;
+		indoubt = fdwxact->indoubt;
+		SpinLockRelease(&fdwxact->mutex);
+
+		values[0] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[1] = ObjectIdGetDatum(fdwxact->serverid);
+		values[2] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (status)
+		{
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[3] = CStringGetTextDatum(xact_status);
+		values[4] = BoolGetDatum(indoubt);
+		values[5] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
+
+/*
+ * Built-in SQL function to resolve a prepared foreign transaction.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			serverid = PG_GETARG_OID(1);
+	Oid			userid = PG_GETARG_OID(2);
+	int			idx;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/* Find and hold the FdwXact entry */
+	idx = get_fdwxact(MyDatabaseId, xid, serverid, userid, true);
+
+	LWLockRelease(FdwXactLock);
+
+	if (idx < 0)
+	{
+		/* No entry */
+		PG_RETURN_BOOL(false);
+	}
+
+	PG_TRY();
+	{
+		FdwXactResolveFdwXacts(&idx, 1, NULL);
+	}
+	PG_CATCH();
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[idx];
+
+		SpinLockAcquire(&fdwxact->mutex);
+		FdwXactCtl->fdwxacts[idx]->held_by = InvalidBackendId;
+		SpinLockRelease(&fdwxact->mutex);
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			serverid = PG_GETARG_OID(1);
+	Oid			userid = PG_GETARG_OID(2);
+	FdwXact		fdwxact;
+	int			i;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->valid && fdwxact->dbid == MyDatabaseId &&
+			fdwxact->local_xid == xid && fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			break;
+	}
+
+	if (i >= FdwXactCtl->num_fdwxacts)
+	{
+		LWLockRelease(FdwXactLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("does not exist foreign transaction on server %u",
+						serverid)));
+	}
+
+	if (fdwxact->inprocessing || fdwxact->held_by != InvalidBackendId)
+	{
+		LWLockRelease(FdwXactLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot remove foreign transaction entry which is being processed")));
+	}
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000000..deb35abfc2
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,566 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool		found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int			slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+		FdwXactRslvCtl->launcher_pid = InvalidPid;
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			memset(resolver, 0, sizeof(FdwXactResolver));
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == InvalidPid);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz now;
+		long		wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int			rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a
+		 * foreign_xact_resolution_retry_interval but always starts when the
+		 * backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool		launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested but not
+			 * running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we should
+			 * retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool		found = false;
+	int			i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the same
+	 * database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find
+		 * FdwXact entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int			unused_slot = -1;
+	int			i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign transaction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to
+	 * wait until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	   *resolver_dbs;	/* DBs resolver's running on */
+	HTAB	   *fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL		ctl;
+	HASH_SEQ_STATUS status;
+	Oid		   *entry;
+	bool		launched;
+	int			i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		bool		indoubt;
+		BackendId	held_by;
+
+		if (!fdwxact->valid)
+			continue;
+
+		SpinLockAcquire(&fdwxact->mutex);
+		indoubt = fdwxact->indoubt;
+		held_by = fdwxact->held_by;
+		SpinLockRelease(&fdwxact->mutex);
+
+		ereport(LOG, (errmsg("[%d] indoubt %s held_by %d id %s",
+							 i,
+							 indoubt ? "true" : "false",
+							 held_by,
+							 fdwxact->fdwxact_id)));
+
+		if ((indoubt && held_by == InvalidBackendId) ||
+			(!indoubt && held_by != InvalidBackendId))
+			hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+	{
+		hash_destroy(fdwxact_dbs);
+		return false;
+	}
+
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool		found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	hash_destroy(fdwxact_dbs);
+	hash_destroy(resolver_dbs);
+
+	return launched;
+}
+
+/* Register a background worker running the foreign transaction launcher */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid			dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int			i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign transaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int			rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		/* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+					   10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000000..b91a2e1e88
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,436 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int			foreign_xact_resolution_retry_interval;
+int			foreign_xact_resolver_timeout = 60 * 1000;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+static void hold_fdwxacts(PGPROC *waiter);
+static void hold_indoubt_fdwxacts(void);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+static TimestampTz last_resolution_time = -1;
+
+/*
+ * held_fdwxacts has indexes of FdwXact which the resolver marked
+ * as in-processing. We clear that flag from those entries on failure.
+ */
+static int *held_fdwxacts = NULL;
+static int	nheld;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	/* clear inprocessing flags */
+	for (int i = 0; i < nheld; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[held_fdwxacts[i]];
+
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwxact->inprocessing = false;
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int			slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+	CommitTransactionCommand();
+
+	held_fdwxacts = palloc(sizeof(int) * max_prepared_foreign_xacts);
+	nheld = 0;
+
+	/* Initialize stats to a sanish value */
+	last_resolution_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TransactionId waitXid = InvalidTransactionId;
+		TimestampTz resolutionTs = -1;
+		TimestampTz now;
+		int			rc;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or the queue has
+		 * only waiters that have a future resolution timestamp.
+		 */
+		for (;;)
+		{
+			PGPROC	   *waiter;
+
+			CHECK_FOR_INTERRUPTS();
+
+			LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+
+			waiter = FdwXactGetWaiter(now, &resolutionTs, &waitXid);
+
+			if (!waiter)
+			{
+				/* Not found, break */
+				LWLockRelease(FdwXactResolutionLock);
+				break;
+			}
+
+			/* Hold the waiting foreign transactions */
+			hold_fdwxacts(waiter);
+			Assert(nheld > 0);
+			LWLockRelease(FdwXactResolutionLock);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveFdwXacts(held_fdwxacts, nheld, waiter);
+			CommitTransactionCommand();
+
+			last_resolution_time = now;
+		}
+
+		/* Hold in-doubt transactions */
+		hold_indoubt_fdwxacts();
+
+		if (nheld > 0)
+		{
+			StartTransactionCommand();
+			FdwXactResolveFdwXacts(held_fdwxacts, nheld, NULL);
+			CommitTransactionCommand();
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(last_resolution_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		/* There is no waiting backend */
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * FdwXactWaiterExists check.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long		sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(last_resolution_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
+
+/*
+ * Mark in-doubt transactions as in-processing.
+ */
+static void
+hold_indoubt_fdwxacts(void)
+{
+	nheld = 0;
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->valid && fdwxact->dbid == MyDatabaseId &&
+			fdwxact->held_by == InvalidBackendId && fdwxact->indoubt)
+		{
+			held_fdwxacts[nheld++] = i;
+
+			/* hold lock */
+			SpinLockAcquire(&fdwxact->mutex);
+			fdwxact->inprocessing = true;
+			SpinLockRelease(&fdwxact->mutex);
+		}
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Mark foreign transactions associated with the given waiter's transaction
+ * as in-processing.
+ */
+static void
+hold_fdwxacts(PGPROC *waiter)
+{
+	nheld = 0;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->valid && fdwxact->dbid == waiter->databaseId &&
+			fdwxact->local_xid == waiter->fdwXactWaitXid)
+		{
+			held_fdwxacts[nheld++] = i;
+
+			/* hold lock */
+			SpinLockAcquire(&fdwxact->mutex);
+			Assert(!fdwxact->indoubt);
+			Assert(fdwxact->held_by = waiter->backendId);
+			fdwxact->inprocessing = true;
+			SpinLockRelease(&fdwxact->mutex);
+		}
+	}
+	LWLockRelease(FdwXactLock);
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index f88d72fd86..982c1a36cc 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -13,6 +13,7 @@ OBJS = \
 	clogdesc.o \
 	committsdesc.o \
 	dbasedesc.o \
+	fdwxactdesc.o \
 	genericdesc.o \
 	gindesc.o \
 	gistdesc.o \
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000000..ca761763e5
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxactdesc.c
+ *		PostgreSQL global transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdwxactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdwxact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		FdwXactOnDiskData *fdwxact_insert = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_insert->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_insert->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_insert->dbid);
+		appendStringInfo(buf, " local xid: %u,", fdwxact_insert->local_xid);
+		appendStringInfo(buf, " id: %s", fdwxact_insert->fdwxact_id);
+	}
+	else
+	{
+		xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_remove->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_remove->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_remove->dbid);
+		appendStringInfo(buf, " local xid: %u", fdwxact_remove->xid);
+	}
+
+}
+
+const char *
+fdwxact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDWXACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDWXACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 1cd97852e8..ea045174e0 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 58091f6b52..200cf9d067 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -10,6 +10,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index e1904877fa..2b9e039580 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -850,6 +851,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 	return result;
 }
 
+/*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
 /*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
@@ -2196,6 +2226,9 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	XLogRecPtr	recptr;
 	TimestampTz committs = GetCurrentTimestamp();
 	bool		replorigin;
+	bool		need_fdwxact_commit;
+
+	need_fdwxact_commit = PrepareFdwXactParticipants(xid);
 
 	/*
 	 * Are we using the replication origins feature?  Or, in other words, are
@@ -2266,6 +2299,16 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	if (need_fdwxact_commit)
+	{
+		SetFdwXactParticipants(xid, true);
+		FdwXactWaitForResolution(xid);
+	}
 }
 
 /*
@@ -2285,6 +2328,9 @@ RecordTransactionAbortPrepared(TransactionId xid,
 							   const char *gid)
 {
 	XLogRecPtr	recptr;
+	bool		need_fdwxact_commit;
+
+	need_fdwxact_commit = PrepareFdwXactParticipants(xid);
 
 	/*
 	 * Catch the scenario where we aborted partway through
@@ -2325,6 +2371,16 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be rolled back.
+	 */
+	if (need_fdwxact_commit)
+	{
+		SetFdwXactParticipants(xid, false);
+		FdwXactWaitForResolution(xid);
+	}
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index cd30b62d36..c611fd8b45 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1219,6 +1220,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1227,6 +1229,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1265,12 +1268,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1428,6 +1432,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitForResolution(xid);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2087,6 +2099,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXact();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2254,6 +2269,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXact(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2341,6 +2357,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXact();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2532,6 +2550,9 @@ PrepareTransaction(void)
 	 */
 	PostPrepare_Twophase();
 
+	/* Release held FdwXact entries */
+	PostPrepare_FdwXact();
+
 	/* PREPARE acts the same as COMMIT as far as GUC is concerned */
 	AtEOXact_GUC(true, 1);
 	AtEOXact_SPI(true);
@@ -2542,6 +2563,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	//AtEOXact_FdwXact(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2751,6 +2773,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXact(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ca09d81b08..eae8c60db3 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
@@ -4599,6 +4600,7 @@ InitControlFile(uint64 sysidentifier)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6286,6 +6288,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_wal_senders",
 									 max_wal_senders,
 									 ControlFile->max_wal_senders);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
@@ -6836,14 +6841,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -7045,7 +7051,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7558,6 +7567,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7888,6 +7898,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9183,6 +9196,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9712,8 +9726,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9731,6 +9747,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9747,6 +9764,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9952,6 +9970,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -10151,6 +10170,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 56420bbc9d..56af9e6408 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -333,6 +333,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 6d53dc463c..a1dea253c2 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2807,8 +2807,14 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
+	}
 
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index f197869752..6206265424 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1101,6 +1103,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (FdwXactExists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1419,6 +1433,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
+	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (FdwXactExists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
 	/*
 	 * Do the deletion
 	 */
@@ -1572,6 +1595,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index fb6ce49056..3fa8bfe09f 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/table.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
@@ -939,7 +940,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 513471ab9b..29f376e48c 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,9 +226,31 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
+	}
 
 	return scanstate;
 }
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 1ec07bad07..e5dee94764 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -2418,6 +2420,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index 61e48ca3f8..8f411c0559 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMappingOid - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,18 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction && !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction && routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper must support both commit and rollback routines or neither");
+
+	if (routine->PrepareForeignTransaction &&
+		!routine->CommitForeignTransaction &&
+		!routine->RollbackForeignTransaction)
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index beb5e85434..2258424e81 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -12,6 +12,8 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -128,6 +130,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index d7f99d9944..84bb1913f3 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3667,6 +3667,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3777,6 +3783,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_EXECUTE_GATHER:
 			event_name = "ExecuteGather";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 		case WAIT_EVENT_HASH_BATCH_ALLOCATE:
 			event_name = "HashBatchAllocate";
 			break;
@@ -4103,6 +4112,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 160afe9f39..6a83f19e24 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -909,6 +911,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -973,12 +979,13 @@ PostmasterMain(int argc, char *argv[])
 	LocalProcessControlFile(false);
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index c2e5e3abf8..9d34817f39 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -151,6 +151,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 427b0d59cd..55609eed81 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(void)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(void)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 3c2b369615..56c43cf741 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -94,6 +94,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -249,6 +251,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1311,6 +1314,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1376,6 +1380,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1425,6 +1430,15 @@ GetOldestXmin(Relation rel, int flags)
 		NormalTransactionIdPrecedes(replication_slot_xmin, result))
 		result = replication_slot_xmin;
 
+	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
 	/*
 	 * After locks have been released and vacuum_defer_cleanup_age has been
 	 * applied, check whether we need to back up further to make logical
@@ -3125,6 +3139,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations of the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index e6985e8eed..241b099238 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -50,3 +50,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 XactTruncationLock					44
+FdwXactLock							45
+FdwXactResolverLock					46
+FdwXactResolutionLock				47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index f5eef6fa4e..9bd1e1791a 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -421,6 +422,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* Initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -822,6 +827,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 8958ec8103..5ed6c05b18 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3054,6 +3056,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 2f3e0a70e0..1e46aff23f 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -426,6 +427,25 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 	{NULL, 0, false}
 };
 
+/*
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
 /*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
@@ -763,6 +783,10 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FOREIGN_TRANSACTION */
+	gettext_noop("Foreign Transaction"),
+	/* FOREIGN_TRANSACTION_RESOLVER */
+	gettext_noop("Foreign Transaction / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2471,6 +2495,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FOREIGN_TRANSACTION_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FOREIGN_TRANSACTION_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4599,6 +4669,16 @@ static struct config_enum ConfigureNamesEnum[] =
 		NULL, assign_synchronous_commit, NULL
 	},
 
+	{
+		{"foreign_twophase_commit", PGC_USERSET, FOREIGN_TRANSACTION,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 995b6ca155..d7ca008a9e 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -344,6 +346,20 @@
 #max_sync_workers_per_subscription = 2	# taken from max_logical_replication_workers
 
 
+#------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+#foreign_twophase_commit = disabled	# use two-phase commit for distributed transactions:
+					# disabled, prefer or required
+
 #------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index a0b0458108..8701c5f005 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 67021a6dc1..78d882ddb2 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -208,6 +208,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index e73639df74..3041c39bc0 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -302,6 +302,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 233441837f..b040202043 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -710,6 +710,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -914,6 +915,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/fdwxactdesc.c b/src/bin/pg_waldump/fdwxactdesc.c
new file mode 120000
index 0000000000..ce8c21880c
--- /dev/null
+++ b/src/bin/pg_waldump/fdwxactdesc.c
@@ -0,0 +1 @@
+../../../src/backend/access/rmgrdesc/fdwxactdesc.c
\ No newline at end of file
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca4b1..b616cea347 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000000..32e041e246
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,163 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL global transaction manager
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDWXACT_H
+#define FDWXACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* fdwXactState */
+#define	FDWXACT_NOT_WAITING		0
+#define	FDWXACT_WAITING			1
+#define	FDWXACT_WAIT_COMPLETE	2
+
+/* Flag passed to FDW transaction management APIs */
+#define FDWXACT_FLAG_ONEPHASE		0x01	/* transaction can commit/rollback
+											 * without preparation */
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER, /* use twophase commit where available */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support
+										 * twophase commit */
+}			ForeignTwophaseCommitLevel;
+
+/* Enum to track the status of foreign transaction */
+typedef enum
+{
+	FDWXACT_STATUS_INVALID = 0,
+	FDWXACT_STATUS_PREPARING,	/* foreign transaction is being prepared */
+	FDWXACT_STATUS_PREPARED,	/* foreign transaction is prepared */
+	FDWXACT_STATUS_COMMITTING,	/* foreign prepared transaction is being
+								 * committed */
+	FDWXACT_STATUS_ABORTING		/* foreign prepared transaction is being
+								 * aborted */
+} FdwXactStatus;
+
+typedef struct FdwXactData *FdwXact;
+
+/*
+ * Shared memory state of a single foreign transaction.
+ */
+typedef struct FdwXactData
+{
+	FdwXact		fdwxact_free_next;	/* Next free FdwXact entry */
+
+	TransactionId local_xid;	/* XID of local transaction */
+	Oid			dbid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			umid;
+
+	FdwXactStatus status;
+	bool		indoubt;		/* Is an in-doubt transaction? */
+	bool		inprocessing;	/* resolver is processing? */
+	slock_t		mutex;			/* protect above three fields */
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;	/* XLOG offset of inserting this entry
+									 * start */
+	XLogRecPtr	insert_end_lsn; /* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to
+								 * file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* prepared transaction
+												 * identifier */
+}			FdwXactData;
+
+/*
+ * Shared memory layout for maintaining foreign prepared transaction entries.
+ * Adding or removing FdwXact entry needs to hold FdwXactLock in exclusive mode,
+ * and iterating fdwXacts needs that in shared mode.
+ */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		free_fdwxacts;
+
+	/* Number of valid foreign transaction entries */
+	int			num_fdwxacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdwxacts[FLEXIBLE_ARRAY_MEMBER];	/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* State data for foreign transaction resolution, passed to FDW callbacks */
+typedef struct FdwXactRslvState
+{
+	TransactionId xid;
+
+	/* Foreign transaction information */
+	char	   *fdwxact_id;
+	ForeignServer *server;
+	UserMapping *usermapping;
+
+	int			flags;			/* OR of FDWXACT_FLAG_xx flags */
+} FdwXactRslvState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+/* Function declarations */
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void RegisterFdwXactByRelId(Oid relid, bool modified);
+extern void RegisterFdwXactByServerId(Oid serverid, bool modified);
+extern void FdwXactReleaseWaiter(PGPROC *waiter);
+extern void FdwXactWaitForResolution(TransactionId wait_xid);
+extern void FdwXactResolveFdwXacts(int *fdwxact_idxs, int nfdwxacts, PGPROC *waiter);
+extern PGPROC *FdwXactGetWaiter(TimestampTz now, TimestampTz *nextResolutionTs_p,
+								TransactionId *waitXid_p);
+extern bool FdwXactWaiterExists(Oid dbid);
+extern bool PrepareFdwXactParticipants(TransactionId xid);
+extern void SetFdwXactParticipants(TransactionId xid, bool commit);
+extern void PreCommit_FdwXact(void);
+extern void AtEOXact_FdwXact(bool is_commit);
+extern void AtPrepare_FdwXact(void);
+extern void PostPrepare_FdwXact(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern bool FdwXactExists(Oid dboid, Oid serverid, Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+
+#endif							/* FDWXACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000000..688b43b8d0
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,28 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherRequestToLaunch(void);
+extern void FdwXactLaunchOrWakeupResolver(void);
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+
+
+#endif							/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000000..779848113c
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int	foreign_xact_resolver_timeout;
+
+#endif							/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000000..b4cec76eae
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,54 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDWXACT_INSERT	0x00
+#define XLOG_FDWXACT_REMOVE	0x10
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDWXACT_ID_MAX_LEN 200
+
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdwxact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+	bool		force;
+} xl_fdwxact_remove;
+
+extern void fdwxact_redo(XLogReaderState *record);
+extern void fdwxact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdwxact_identify(uint8 info);
+
+#endif							/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000000..c935471936
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,63 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/resolver_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t		pid;			/* this resolver's PID, or 0 if not active */
+	Oid			dbid;			/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool		in_use;
+
+	/* Protect shared variables shown above */
+	slock_t		mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	   *latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/* Foreign transaction resolution queue. Protected by FdwXactLock */
+	SHM_QUEUE	fdwxact_queue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch	   *launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+#define SizeOfFdwXactRslvCtlData \
+	(offsetof(FdwXactRslvCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif							/* RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 6c15df7e70..986bc73566 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Fdw Transaction", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 2ca71c3445..bd027a2861 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index a04fc70326..6f1f336e31 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -108,6 +108,13 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
+/*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index c8869d5226..da0d442f1b 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -232,6 +232,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index de5670e538..9884f5f8e7 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..df5189dd2d 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5981,6 +5981,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '9706', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o}',
+  proargnames => '{xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '9707', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '9708', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -6099,6 +6117,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '9709',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 95556dfb15..8d046cc4e4 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 5e0cf533fb..5596ee591c 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -69,6 +69,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 											   bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 														 bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index c55dc1481c..2186c1c5d0 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -806,6 +806,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -853,6 +855,7 @@ typedef enum
 	WAIT_EVENT_CHECKPOINT_DONE,
 	WAIT_EVENT_CHECKPOINT_START,
 	WAIT_EVENT_EXECUTE_GATHER,
+	WAIT_EVENT_FDWXACT_RESOLUTION,
 	WAIT_EVENT_HASH_BATCH_ALLOCATE,
 	WAIT_EVENT_HASH_BATCH_ELECT,
 	WAIT_EVENT_HASH_BATCH_LOAD,
@@ -969,6 +972,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index 1ee9000b2b..4150d8a3e4 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -16,6 +16,7 @@
 
 #include "access/clog.h"
 #include "access/xlogdefs.h"
+#include "datatype/timestamp.h"
 #include "lib/ilist.h"
 #include "storage/latch.h"
 #include "storage/lock.h"
@@ -154,6 +155,16 @@ struct PGPROC
 	int			syncRepState;	/* wait state for sync rep */
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
+	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
 	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index a5c7d0c064..0f73b64937 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -125,4 +127,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 454c2df487..f977ca43d4 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,8 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FOREIGN_TRANSACTION,
+	FOREIGN_TRANSACTION_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..628eaf531e 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1342,6 +1342,13 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
-- 
2.23.0

From bdadc8426a40ab0c1abb427b27c48e1dff316d18 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Wed, 3 Jun 2020 16:38:13 +0900
Subject: [PATCH v22 6/6] Add prefer mode to foreign_twophase_commit.

---
 doc/src/sgml/config.sgml                      | 33 ++++++++--------
 doc/src/sgml/distributed-transaction.sgml     | 10 ++---
 src/backend/access/fdwxact/fdwxact.c          | 38 ++++++++++++++++---
 src/backend/utils/misc/guc.c                  |  5 ++-
 src/backend/utils/misc/postgresql.conf.sample |  2 +-
 src/include/access/fdwxact.h                  |  1 +
 .../test_fdwxact/expected/test_fdwxact.out    | 25 ++++++++++++
 .../modules/test_fdwxact/sql/test_fdwxact.sql | 28 ++++++++++++++
 8 files changed, 114 insertions(+), 28 deletions(-)

diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 918fac967c..18f3e967fe 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -9102,18 +9102,21 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
         <para>
          Specifies whether distributed transaction commits ensures that all
          involved changes on foreign servers are committed or not. Valid
-         values are <literal>required</literal> and <literal>disabled</literal>.
-         The default setting is <literal>disabled</literal>. Setting to
-         <literal>disabled</literal> don't use two-phase commit protocol to
-         commit or rollback distributed transactions. When set to
-         <literal>required</literal> distributed transactions strictly requires
-         that all written servers can use two-phase commit protocol.  That is,
-         the distributed transaction cannot commit if even one server does not
-         support the prepare callback routine
+         values are <literal>required</literal>, <literal>prefer</literal> and
+         <literal>disabled</literal>. The default setting is
+         <literal>disabled</literal>. Setting to <literal>disabled</literal>
+         don't use two-phase commit protocol to commit or rollback distributed
+         transactions. When set to <literal>required</literal> distributed
+         transactions strictly requires that all written servers can use
+         two-phase commit protocol.  That is, the distributed transaction cannot
+         commit if even one server does not support the prepare callback routine
          (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
-         In <literal>required</literal> case, distributed transaction commit will
-         wait for all involving foreign transaction to be committed before the
-         command return a "success" indication to the client.
+         When set to <literal>prefer</literal> the distributed transaction use
+         two-phase commit protocol on only servers where available and commit on
+         others. In <literal>prefer</literal> and <literal>required</literal> case,
+         distributed transaction commit will wait for all involving foreign
+         transaction to be committed before the command return a "success"
+         indication to the client.
         </para>
 
         <para>
@@ -9123,10 +9126,10 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
 
         <note>
          <para>
-          When <literal>disabled</literal> there can be risk of database
-          consistency among all servers that involved in the distributed
-          transaction when some foreign server crashes during committing the
-          distributed transaction.
+          When <literal>disabled</literal> or <literal>prefer</literal> there
+          can be risk of database consistency among all servers that involved in
+          the distributed transaction when some foreign server crashes during
+          committing the distributed transaction.
          </para>
         </note>
        </listitem>
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
index 139dd7f918..d992a807c3 100644
--- a/doc/src/sgml/distributed-transaction.sgml
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -48,11 +48,11 @@
        prepares all transaction on the foreign servers if two-phase commit is
        required. Two-phase commit is required when the transaction modifies
        data on two or more servers including the local server itself and
-       <xref linkend="guc-foreign-twophase-commit"/> is
-       <literal>required</literal>. If all preparations on foreign servers got
-       successful go to the next step. Any failure happens in this step,
-       the server changes to rollback, then rollback all transactions on both
-       local and foreign servers.
+       <xref linkend="guc-foreign-twophase-commit"/> is either
+       <literal>required</literal> or <literal>prefer</literal>.  If all
+       preparations on foreign servers got successful go to the next step.
+       Any failure happens in this step, the server changes to rollback,
+       then rollback all transactions on both local and foreign servers.
       </para>
      </listitem>
      <listitem>
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
index 86630972c0..b29b463344 100644
--- a/src/backend/access/fdwxact/fdwxact.c
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -441,7 +441,9 @@ create_fdwxact_participant(Oid serverid, Oid userid, FdwRoutine *routine)
  * When foreign twophase commit is enabled, the behavior depends on the value
  * of foreign_twophase_commit; when 'required' we strictly require for all
  * foreign servers' FDW to support two-phase commit protocol and ask them to
- * prepare foreign transactions, and when 'disabled' we ask all foreign servers
+ * prepare foreign transactions, when 'prefer' we ask only foreign servers
+ * that are capable of two-phase commit to prepare foreign transactions and ask
+ * for other servers to commit, and when 'disabled' we ask all foreign servers
  * to commit foreign transaction in one-phase. If we failed to commit any of
  * them we change to aborting.
  *
@@ -536,8 +538,9 @@ checkForeignTwophaseCommitRequired(void)
 {
 	ListCell   *lc;
 	bool		need_twophase_commit;
-	bool		have_notwophase = false;
+	bool		have_notwophase;
 	int			nserverswritten = 0;
+	int			nserverstwophase = 0;
 
 	if (!IsForeignTwophaseCommitRequested())
 		return false;
@@ -549,16 +552,29 @@ checkForeignTwophaseCommitRequired(void)
 		if (!fdw_part->modified)
 			continue;
 
-		if (!SeverSupportTwophaseCommit(fdw_part))
-			have_notwophase = true;
+		if (SeverSupportTwophaseCommit(fdw_part))
+			nserverstwophase++;
 
 		nserverswritten++;
 	}
+	Assert(nserverswritten >= nserverstwophase);
+
+	/* check if there is any servers that don't support two-phase commit */
+	have_notwophase = (nserverswritten != nserverstwophase);
 
 	/* Did we modify the local non-temporary data? */
 	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+	{
 		nserverswritten++;
 
+		/*
+		 * We increment nserverstwophase as well for making code simple,
+		 * although we don't actually use two-phase commit for the local
+		 * transaction.
+		 */
+		nserverstwophase++;
+	}
+
 	if (nserverswritten <= 1)
 		return false;
 
@@ -570,6 +586,17 @@ checkForeignTwophaseCommitRequired(void)
 		 */
 		need_twophase_commit = (nserverswritten >= 2);
 	}
+	else
+	{
+		Assert(foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_PREFER);
+
+		/*
+		 * In 'prefer' case, we use two-phase commit when this transaction modified
+		 * two or more servers including the local server or servers that support
+		 * two-phase commit.
+		 */
+		need_twophase_commit = (nserverstwophase >= 2);
+	}
 
 	/*
 	 * If foreign two phase commit is required then all foreign serves must be
@@ -590,7 +617,8 @@ checkForeignTwophaseCommitRequired(void)
 					 errmsg("foreign two-phase commit is required but prepared foreign transactions are disabled"),
 					 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
 
-		if (have_notwophase)
+		if (have_notwophase &&
+			foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED)
 			ereport(ERROR,
 					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
 					 errmsg("cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol"),
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 313bf33324..1e46aff23f 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -428,11 +428,12 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 };
 
 /*
- * Although only "required" and "disabled" are documented, we accept all
- * the likely variants of "on" and "off".
+ * Although only "required", "prefer", and "disabled" are documented,
+ *  we accept all the likely variants of "on" and "off".
  */
 static const struct config_enum_entry foreign_twophase_commit_options[] = {
 	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"prefer", FOREIGN_TWOPHASE_COMMIT_PREFER, false},
 	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
 	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
 	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 03de9fbc81..47bb402680 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -358,7 +358,7 @@
 							# foreign transactions
 							# after a failed attempt
 #foreign_twophase_commit = disabled	# use two-phase commit for distributed transactions:
-					# disabled or required
+					# disabled, prefer or required
 
 #------------------------------------------------------------------------------
 # QUERY TUNING
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
index 5df32b2703..32e041e246 100644
--- a/src/include/access/fdwxact.h
+++ b/src/include/access/fdwxact.h
@@ -37,6 +37,7 @@
 typedef enum
 {
 	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_PREFER, /* use twophase commit where available */
 	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support
 										 * twophase commit */
 }			ForeignTwophaseCommitLevel;
diff --git a/src/test/modules/test_fdwxact/expected/test_fdwxact.out b/src/test/modules/test_fdwxact/expected/test_fdwxact.out
index c6a91ac9f1..ce8465b52c 100644
--- a/src/test/modules/test_fdwxact/expected/test_fdwxact.out
+++ b/src/test/modules/test_fdwxact/expected/test_fdwxact.out
@@ -221,3 +221,28 @@ BEGIN;
 INSERT INTO ft_1 VALUES (1);
 PREPARE TRANSACTION 'global_x1';
 ERROR:  cannot PREPARE a distributed transaction when foreign_twophase_commit is 'disabled'
+-- Test 'prefer' mode.
+-- The cases where failed in 'required' mode should pass in 'prefer' mode.
+-- We simply commit/rollback a transaction in one-phase on a server
+-- that doesn't support two-phase commit, instead of error.
+SET foreign_twophase_commit TO 'prefer';
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
diff --git a/src/test/modules/test_fdwxact/sql/test_fdwxact.sql b/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
index 8cf860e295..72a9ee6be4 100644
--- a/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
+++ b/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
@@ -191,3 +191,31 @@ PREPARE TRANSACTION 'global_x1';
 BEGIN;
 INSERT INTO ft_1 VALUES (1);
 PREPARE TRANSACTION 'global_x1';
+
+
+-- Test 'prefer' mode.
+-- The cases where failed in 'required' mode should pass in 'prefer' mode.
+-- We simply commit/rollback a transaction in one-phase on a server
+-- that doesn't support two-phase commit, instead of error.
+SET foreign_twophase_commit TO 'prefer';
+
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
-- 
2.23.0

From 6c2e3227c020de105dfa071f478e9534f6cb24eb Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 21:41:29 +0500
Subject: [PATCH v22 5/6] Add regression tests for foreign twophase commit.

---
 src/test/modules/Makefile                     |   1 +
 src/test/modules/test_fdwxact/.gitignore      |   4 +
 src/test/modules/test_fdwxact/Makefile        |  28 ++
 .../test_fdwxact/expected/test_fdwxact.out    | 223 +++++++++
 src/test/modules/test_fdwxact/fdwxact.conf    |   7 +
 .../modules/test_fdwxact/sql/test_fdwxact.sql | 193 +++++++
 src/test/modules/test_fdwxact/t/001_basic.pl  | 137 +++++
 .../test_fdwxact/test_fdwxact--1.0.sql        |  44 ++
 src/test/modules/test_fdwxact/test_fdwxact.c  | 471 ++++++++++++++++++
 .../modules/test_fdwxact/test_fdwxact.control |   4 +
 src/test/recovery/Makefile                    |   2 +-
 src/test/recovery/t/021_fdwxact.pl            | 175 +++++++
 src/test/regress/pg_regress.c                 |  13 +-
 13 files changed, 1297 insertions(+), 5 deletions(-)
 create mode 100644 src/test/modules/test_fdwxact/.gitignore
 create mode 100644 src/test/modules/test_fdwxact/Makefile
 create mode 100644 src/test/modules/test_fdwxact/expected/test_fdwxact.out
 create mode 100644 src/test/modules/test_fdwxact/fdwxact.conf
 create mode 100644 src/test/modules/test_fdwxact/sql/test_fdwxact.sql
 create mode 100644 src/test/modules/test_fdwxact/t/001_basic.pl
 create mode 100644 src/test/modules/test_fdwxact/test_fdwxact--1.0.sql
 create mode 100644 src/test/modules/test_fdwxact/test_fdwxact.c
 create mode 100644 src/test/modules/test_fdwxact/test_fdwxact.control
 create mode 100644 src/test/recovery/t/021_fdwxact.pl

diff --git a/src/test/modules/Makefile b/src/test/modules/Makefile
index 29de73c060..8a48e6ba19 100644
--- a/src/test/modules/Makefile
+++ b/src/test/modules/Makefile
@@ -13,6 +13,7 @@ SUBDIRS = \
 		  test_bloomfilter \
 		  test_ddl_deparse \
 		  test_extensions \
+		  test_fdwxact \
 		  test_ginpostinglist \
 		  test_integerset \
 		  test_misc \
diff --git a/src/test/modules/test_fdwxact/.gitignore b/src/test/modules/test_fdwxact/.gitignore
new file mode 100644
index 0000000000..5dcb3ff972
--- /dev/null
+++ b/src/test/modules/test_fdwxact/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/test_fdwxact/Makefile b/src/test/modules/test_fdwxact/Makefile
new file mode 100644
index 0000000000..b3fc99aee3
--- /dev/null
+++ b/src/test/modules/test_fdwxact/Makefile
@@ -0,0 +1,28 @@
+# src/test/modules/test_fdwxact/Makefile
+
+MODULE_big = test_fdwxact
+OBJS = \
+	$(WIN32RES) \
+	test_fdwxact.o
+PGFILEDESC = "test_fdwxact - test code for src/backend/access/fdwxact"
+
+EXTENSION = test_fdwxact
+DATA = test_fdwxact--1.0.sql
+
+REGRESS_OPTS = --temp-config=$(top_srcdir)/src/test/modules/test_fdwxact/fdwxact.conf
+REGRESS = test_fdwxact
+
+NO_INSTALLCHECK = 1
+
+TAP_TESTS =1
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_fdwxact/expected/test_fdwxact.out b/src/test/modules/test_fdwxact/expected/test_fdwxact.out
new file mode 100644
index 0000000000..c6a91ac9f1
--- /dev/null
+++ b/src/test/modules/test_fdwxact/expected/test_fdwxact.out
@@ -0,0 +1,223 @@
+--
+-- Test for foreign transaction management.
+--
+CREATE EXTENSION test_fdwxact;
+-- setup two servers that don't support transaction management API
+CREATE SERVER srv_1 FOREIGN DATA WRAPPER test_fdw;
+CREATE SERVER srv_2 FOREIGN DATA WRAPPER test_fdw;
+-- setup two servers that support only commit and rollback API
+CREATE SERVER srv_no2pc_1 FOREIGN DATA WRAPPER test_no2pc_fdw;
+CREATE SERVER srv_no2pc_2 FOREIGN DATA WRAPPER test_no2pc_fdw;
+-- setup two servers that support commit, rollback and prepare API.
+-- That is, those two server support two-phase commit protocol.
+CREATE SERVER srv_2pc_1 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE SERVER srv_2pc_2 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE TABLE t (i int);
+CREATE FOREIGN TABLE ft_1 (i int) SERVER srv_1;
+CREATE FOREIGN TABLE ft_2 (i int) SERVER srv_2;
+CREATE FOREIGN TABLE ft_no2pc_1 (i int) SERVER srv_no2pc_1;
+CREATE FOREIGN TABLE ft_no2pc_2 (i int) SERVER srv_no2pc_2;
+CREATE FOREIGN TABLE ft_2pc_1 (i int) SERVER srv_2pc_1;
+CREATE FOREIGN TABLE ft_2pc_2 (i int) SERVER srv_2pc_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_2;
+-- Test 'disabled' mode.
+-- Modifies one or two servers but since we don't require two-phase
+-- commit, all case should not raise an error.
+SET foreign_twophase_commit TO disabled;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+ROLLBACK;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+ROLLBACK;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+ROLLBACK;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+-- Test 'required' mode.
+-- In this case, when two-phase commit is required, all servers
+-- which are involved in the and modified need to support two-phase
+-- commit protocol. Otherwise transaction will rollback.
+SET foreign_twophase_commit TO 'required';
+-- Ok. Writing only one server doesn't require two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+-- Ok. Writing two servers, we require two-phase commit and success.
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+-- Ok. Only reading servers doesn't require two-phase commit.
+BEGIN;
+SELECT * FROM ft_2pc_1;
+ i 
+---
+(0 rows)
+
+SELECT * FROM ft_2pc_2;
+ i 
+---
+(0 rows)
+
+COMMIT;
+BEGIN;
+SELECT * FROM ft_1;
+ i 
+---
+(0 rows)
+
+SELECT * FROM ft_no2pc_1;
+ i 
+---
+(0 rows)
+
+COMMIT;
+-- Ok. Read one server and write one server.
+BEGIN;
+SELECT * FROM ft_1;
+ i 
+---
+(0 rows)
+
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+SELECT * FROM ft_no2pc_1;
+ i 
+---
+(0 rows)
+
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+-- Error. ft_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. ft_no2pc_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. Both ft_1 and ft_2 don't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. Both ft_no2pc_1 and ft_no2pc_2 don't support two-phase
+-- commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Error. Two-phase commit is required because of writes on two
+-- servers: local node and ft_no2pc_1. But ft_no2pc_1 doesn't support
+-- two-phase commit.
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+ERROR:  cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol
+DETAIL:  foreign_twophase_commit is 'required' but the transaction has some foreign servers which are not capable of two-phase commit
+-- Tests for PREPARE.
+-- Prepare two transactions: local and foreign.
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+INSERT INTO t VALUES(3);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*) FROM pg_foreign_xacts();
+ count 
+-------
+     1
+(1 row)
+
+COMMIT PREPARED 'global_x1';
+SELECT count(*) FROM pg_foreign_xacts();
+ count 
+-------
+     0
+(1 row)
+
+-- Even if the transaction modified only one foreign server,
+-- we prepare foreign transaction.
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*) FROM pg_foreign_xacts();
+ count 
+-------
+     1
+(1 row)
+
+ROLLBACK PREPARED 'global_x1';
+SELECT count(*) FROM pg_foreign_xacts();
+ count 
+-------
+     0
+(1 row)
+
+-- Error. PREPARE needs all involved foreign servers to
+-- support two-phsae commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+ERROR:  cannot PREPARE a distributed transaction which has operated on a foreign server not supporting two-phase commit protocol
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+ERROR:  cannot PREPARE a distributed transaction which has operated on a foreign server not supporting two-phase commit protocol
+-- Error. We cannot PREPARE a distributed transaction when
+-- foreign_twophase_commit is disabled.
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+ERROR:  cannot PREPARE a distributed transaction when foreign_twophase_commit is 'disabled'
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+ERROR:  cannot PREPARE a distributed transaction when foreign_twophase_commit is 'disabled'
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+ERROR:  cannot PREPARE a distributed transaction when foreign_twophase_commit is 'disabled'
diff --git a/src/test/modules/test_fdwxact/fdwxact.conf b/src/test/modules/test_fdwxact/fdwxact.conf
new file mode 100644
index 0000000000..20e4a671df
--- /dev/null
+++ b/src/test/modules/test_fdwxact/fdwxact.conf
@@ -0,0 +1,7 @@
+shared_preload_libraries = 'test_fdwxact'
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 1
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = disabled
diff --git a/src/test/modules/test_fdwxact/sql/test_fdwxact.sql b/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
new file mode 100644
index 0000000000..8cf860e295
--- /dev/null
+++ b/src/test/modules/test_fdwxact/sql/test_fdwxact.sql
@@ -0,0 +1,193 @@
+--
+-- Test for foreign transaction management.
+--
+
+CREATE EXTENSION test_fdwxact;
+
+-- setup two servers that don't support transaction management API
+CREATE SERVER srv_1 FOREIGN DATA WRAPPER test_fdw;
+CREATE SERVER srv_2 FOREIGN DATA WRAPPER test_fdw;
+
+-- setup two servers that support only commit and rollback API
+CREATE SERVER srv_no2pc_1 FOREIGN DATA WRAPPER test_no2pc_fdw;
+CREATE SERVER srv_no2pc_2 FOREIGN DATA WRAPPER test_no2pc_fdw;
+
+-- setup two servers that support commit, rollback and prepare API.
+-- That is, those two server support two-phase commit protocol.
+CREATE SERVER srv_2pc_1 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE SERVER srv_2pc_2 FOREIGN DATA WRAPPER test_2pc_fdw;
+
+CREATE TABLE t (i int);
+CREATE FOREIGN TABLE ft_1 (i int) SERVER srv_1;
+CREATE FOREIGN TABLE ft_2 (i int) SERVER srv_2;
+CREATE FOREIGN TABLE ft_no2pc_1 (i int) SERVER srv_no2pc_1;
+CREATE FOREIGN TABLE ft_no2pc_2 (i int) SERVER srv_no2pc_2;
+CREATE FOREIGN TABLE ft_2pc_1 (i int) SERVER srv_2pc_1;
+CREATE FOREIGN TABLE ft_2pc_2 (i int) SERVER srv_2pc_2;
+
+CREATE USER MAPPING FOR PUBLIC SERVER srv_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc_2;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_2;
+
+
+-- Test 'disabled' mode.
+-- Modifies one or two servers but since we don't require two-phase
+-- commit, all case should not raise an error.
+SET foreign_twophase_commit TO disabled;
+
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+ROLLBACK;
+
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+ROLLBACK;
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+ROLLBACK;
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+
+
+-- Test 'required' mode.
+-- In this case, when two-phase commit is required, all servers
+-- which are involved in the and modified need to support two-phase
+-- commit protocol. Otherwise transaction will rollback.
+SET foreign_twophase_commit TO 'required';
+
+-- Ok. Writing only one server doesn't require two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Ok. Writing two servers, we require two-phase commit and success.
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+INSERT INTO ft_2pc_2 VALUES (1);
+COMMIT;
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Ok. Only reading servers doesn't require two-phase commit.
+BEGIN;
+SELECT * FROM ft_2pc_1;
+SELECT * FROM ft_2pc_2;
+COMMIT;
+BEGIN;
+SELECT * FROM ft_1;
+SELECT * FROM ft_no2pc_1;
+COMMIT;
+
+-- Ok. Read one server and write one server.
+BEGIN;
+SELECT * FROM ft_1;
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+BEGIN;
+SELECT * FROM ft_no2pc_1;
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Error. ft_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Error. ft_no2pc_1 doesn't support two-phase commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_2pc_1 VALUES (1);
+COMMIT;
+
+-- Error. Both ft_1 and ft_2 don't support two-phase commit.
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+INSERT INTO ft_2 VALUES (1);
+COMMIT;
+
+-- Error. Both ft_no2pc_1 and ft_no2pc_2 don't support two-phase
+-- commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+INSERT INTO ft_no2pc_2 VALUES (1);
+COMMIT;
+
+-- Error. Two-phase commit is required because of writes on two
+-- servers: local node and ft_no2pc_1. But ft_no2pc_1 doesn't support
+-- two-phase commit.
+BEGIN;
+INSERT INTO t VALUES (1);
+INSERT INTO ft_no2pc_1 VALUES (1);
+COMMIT;
+
+
+-- Tests for PREPARE.
+-- Prepare two transactions: local and foreign.
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+INSERT INTO t VALUES(3);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*) FROM pg_foreign_xacts();
+COMMIT PREPARED 'global_x1';
+SELECT count(*) FROM pg_foreign_xacts();
+
+-- Even if the transaction modified only one foreign server,
+-- we prepare foreign transaction.
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES(1);
+PREPARE TRANSACTION 'global_x1';
+SELECT count(*) FROM pg_foreign_xacts();
+ROLLBACK PREPARED 'global_x1';
+SELECT count(*) FROM pg_foreign_xacts();
+
+-- Error. PREPARE needs all involved foreign servers to
+-- support two-phsae commit.
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+
+-- Error. We cannot PREPARE a distributed transaction when
+-- foreign_twophase_commit is disabled.
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft_2pc_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+BEGIN;
+INSERT INTO ft_no2pc_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
+BEGIN;
+INSERT INTO ft_1 VALUES (1);
+PREPARE TRANSACTION 'global_x1';
diff --git a/src/test/modules/test_fdwxact/t/001_basic.pl b/src/test/modules/test_fdwxact/t/001_basic.pl
new file mode 100644
index 0000000000..8d48a74e86
--- /dev/null
+++ b/src/test/modules/test_fdwxact/t/001_basic.pl
@@ -0,0 +1,137 @@
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 11;
+
+my $node = get_new_node('main');
+$node->init;
+$node->append_conf('postgresql.conf', qq(
+shared_preload_libraries = 'test_fdwxact'
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = required
+test_fdwxact.log_api_calls = true
+				   ));
+$node->start;
+
+$node->psql(
+	'postgres', "
+CREATE EXTENSION test_fdwxact;
+CREATE SERVER srv FOREIGN DATA WRAPPER test_fdw;
+CREATE SERVER srv_no2pc FOREIGN DATA WRAPPER test_no2pc_fdw;
+CREATE SERVER srv_2pc_1 FOREIGN DATA WRAPPER test_2pc_fdw;
+CREATE SERVER srv_2pc_2 FOREIGN DATA WRAPPER test_2pc_fdw;
+
+CREATE TABLE t (i int);
+CREATE FOREIGN TABLE ft (i int) SERVER srv;
+CREATE FOREIGN TABLE ft_no2pc (i int) SERVER srv_no2pc;
+CREATE FOREIGN TABLE ft_2pc_1 (i int) SERVER srv_2pc_1;
+CREATE FOREIGN TABLE ft_2pc_2 (i int) SERVER srv_2pc_2;
+
+CREATE USER MAPPING FOR PUBLIC SERVER srv;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_no2pc;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_1;
+CREATE USER MAPPING FOR PUBLIC SERVER srv_2pc_2;
+	");
+
+sub run_transaction
+{
+	my ($node, $prepsql, $sql, $endsql) = @_;
+
+	$endsql = 'COMMIT' unless defined $endsql;
+
+	local $ENV{PGHOST} = $node->host;
+	local $ENV{PGPORT} = $node->port;
+
+	truncate $node->logfile, 0;
+
+	$node->safe_psql('postgres', $prepsql);
+	my ($cmdret, $stdout, $stderr) = $node->psql('postgres',
+												 "BEGIN;
+												 SELECT txid_current() as xid;
+												 $sql
+												 $endsql;
+												 ");
+	my $log = TestLib::slurp_file($node->logfile);
+
+	return $log, $stdout;
+}
+
+my ($log, $xid);
+
+# The transaction is committed using two-phase commit.
+($log, $xid) = run_transaction($node, "",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+like($log, qr/commit prepared tx_$xid on srv_2pc_1/, "commit prepared transaction-1");
+like($log, qr/commit prepared tx_$xid on srv_2pc_2/, "commit prepared transaction-2");
+
+# Similary, two-phase commit is used.
+($log, $xid) = run_transaction($node, "",
+					  "INSERT INTO t VALUES(1);
+					  INSERT INTO ft_2pc_1 VALUES(1);");
+like($log, qr/commit prepared tx_$xid on srv_2pc_1/, "commit prepared transaction-3");
+
+# Test the case where transaction attempting prepare the local transaction fails after
+# preparing foreign transactions. The first attempt should be succeeded, but the second
+# attempt will fail after preparing foreign transaction, and should rollback the prepared
+# foreign transaction.
+($log, $xid) = run_transaction($node, "",
+							   "INSERT INTO t VALUES(1);
+							   INSERT INTO ft_2pc_1 VALUES(1);",
+							   "PREPARE TRANSACTION 'tx1'");
+($log, $xid) = run_transaction($node, "",
+							   "INSERT INTO t VALUES(1);
+							   INSERT INTO ft_2pc_1 VALUES(1);",
+							   "PREPARE TRANSACTION 'tx1'");
+like($log, qr/rollback prepared tx_$xid on srv_2pc_1/, "failure after prepare transaction");
+$node->safe_psql('postgres', "COMMIT PREPARED 'tx1'");
+
+# Inject an error into prepare phase on srv_2pc_1. The transaction fails during
+# preparing the foreign transaction on srv_2pc_1. Then, we try to both 'rollback' and
+# 'rollback prepared' the foreign transaction, and rollback another foreign
+# transaction.
+($log, $xid) = run_transaction($node,
+							   "SELECT test_inject_error('error', 'prepare', 'srv_2pc_1');",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+like($log, qr/rollback $xid on srv_2pc_1/, "rollback on failed server");
+like($log, qr/rollback prepared tx_$xid on srv_2pc_1/, "rollback prepared on failed server");
+like($log, qr/rollback $xid on srv_2pc_2/, "rollback on another server");
+
+# Inject an panic into prepare phase on srv_2pc_2. The server crashes after preparing both
+# foreign transaction. After the restart, those transactions are recovered as in-doubt
+# transactions. We check if the resolver process rollbacks those transaction after recovery.
+($log, $xid) = run_transaction($node,
+							   "SELECT test_inject_error('panic', 'prepare', 'srv_2pc_2');",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+$node->restart();
+$node->poll_query_until('postgres',
+						"SELECT count(*) = 0 FROM pg_foreign_xacts")
+  or die "Timeout while waiting for resolver process to resolve in-doubt transactions";
+$log = TestLib::slurp_file($node->logfile);
+like($log, qr/rollback prepared tx_[0-9]+ on srv_2pc_1/, "resolver rolled back in-doubt transaction");
+like($log, qr/rollback prepared tx_[0-9]+ on srv_2pc_2/, "resolver rolled back in-doubt transaction");
+truncate $node->logfile, 0;
+
+# Inject an panic into commit phase on srv_2pc_1. The server crashes due to the panic
+# error raised by resolver process during commit prepared foreign transaction on srv_2pc_1.
+# After the restart, those transactions are recovered as in-doubt transactions. We check if
+# the resolver process commits those transaction after recovery.
+($log, $xid) = run_transaction($node,
+							   "SELECT test_inject_error('panic', 'commit', 'srv_2pc_1');",
+							   "INSERT INTO ft_2pc_1 VALUES(1);
+							   INSERT INTO ft_2pc_2 VALUES(1);");
+$node->restart();
+$node->poll_query_until('postgres',
+						"SELECT count(*) = 0 FROM pg_foreign_xacts")
+  or die "Timeout while waiting for resolver process to resolve in-doubt transactions";
+$log = TestLib::slurp_file($node->logfile);
+like($log, qr/commit prepared tx_[0-9]+ on srv_2pc_1/, "resolver rolled back in-doubt transaction");
+like($log, qr/commit prepared tx_[0-9]+ on srv_2pc_2/, "resolver rolled back in-doubt transaction");
+truncate $node->logfile, 0;
diff --git a/src/test/modules/test_fdwxact/test_fdwxact--1.0.sql b/src/test/modules/test_fdwxact/test_fdwxact--1.0.sql
new file mode 100644
index 0000000000..f676dfe04b
--- /dev/null
+++ b/src/test/modules/test_fdwxact/test_fdwxact--1.0.sql
@@ -0,0 +1,44 @@
+/* src/test/modules/test_atomic_commit/test_atomic_commit--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_atomic_commit" to load this file. \quit
+
+-- test_fdw doesn't use transaction API
+CREATE FUNCTION test_fdw_handler()
+RETURNS fdw_handler
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FOREIGN DATA WRAPPER test_fdw
+  HANDLER test_fdw_handler;
+
+-- test_no2pc_fdw uses only COMMIT and ROLLBACK API
+CREATE FUNCTION test_no2pc_fdw_handler()
+RETURNS fdw_handler
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FOREIGN DATA WRAPPER test_no2pc_fdw
+  HANDLER test_no2pc_fdw_handler;
+
+-- test_2pc uses PREPARE API as well
+CREATE FUNCTION test_2pc_fdw_handler()
+RETURNS fdw_handler
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FOREIGN DATA WRAPPER test_2pc_fdw
+  HANDLER test_2pc_fdw_handler;
+
+CREATE FUNCTION test_inject_error(
+elevel text,
+phase text,
+server text)
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
+
+CREATE FUNCTION test_reset_error()
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE C STRICT;
diff --git a/src/test/modules/test_fdwxact/test_fdwxact.c b/src/test/modules/test_fdwxact/test_fdwxact.c
new file mode 100644
index 0000000000..738690c978
--- /dev/null
+++ b/src/test/modules/test_fdwxact/test_fdwxact.c
@@ -0,0 +1,471 @@
+/*-------------------------------------------------------------------------
+ *
+ * test_fdwxact.c
+ *		  Test modules for foreign transaction management
+ *
+ * This module implements three types of foreign data wrapper: the first
+ * doesn't support any transaction FDW APIs, the second supports only
+ * commit and rollback API and the third supports all transaction API including
+ * prepare.
+ *
+ * Also, this module has an ability to inject an error at prepare callback or
+ * commit callback using test_inject_error() SQL function. The information of
+ * injected error is stored in the shared memory so that backend processes and
+ * resolver processes can see it.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		src/test/modules/test_fdwxact/test_fdwxact.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact.h"
+#include "commands/defrem.h"
+#include "access/reloptions.h"
+#include "foreign/fdwapi.h"
+#include "funcapi.h"
+#include "miscadmin.h"
+#include "optimizer/clauses.h"
+#include "optimizer/cost.h"
+#include "optimizer/optimizer.h"
+#include "optimizer/pathnode.h"
+#include "optimizer/paths.h"
+#include "optimizer/planmain.h"
+#include "optimizer/restrictinfo.h"
+#include "storage/ipc.h"
+#include "storage/spin.h"
+#include "utils/builtins.h"
+
+PG_MODULE_MAGIC;
+
+#define TEST_FDWXCT_MAX_NAME_LEN 32
+
+typedef struct testFdwXactSharedState
+{
+	char	elevel[TEST_FDWXCT_MAX_NAME_LEN];
+	char	phase[TEST_FDWXCT_MAX_NAME_LEN];
+	char	server[TEST_FDWXCT_MAX_NAME_LEN];
+	LWLock	*lock;
+} testFdwXactSharedState;
+testFdwXactSharedState *fxss = NULL;
+
+static shmem_startup_hook_type prev_shmem_startup_hook = NULL;
+static bool log_api_calls = false;
+
+void _PG_init(void);
+void _PG_fini(void);
+PG_FUNCTION_INFO_V1(test_fdw_handler);
+PG_FUNCTION_INFO_V1(test_no2pc_fdw_handler);
+PG_FUNCTION_INFO_V1(test_2pc_fdw_handler);
+PG_FUNCTION_INFO_V1(test_inject_error);
+PG_FUNCTION_INFO_V1(test_reset_error);
+
+static void test_fdwxact_shmem_startup(void);
+static bool check_event(char *servername, char *phase, int *elevel);
+static void testGetForeignRelSize(PlannerInfo *root,
+								  RelOptInfo *baserel,
+								  Oid foreigntableid);
+static void testGetForeignPaths(PlannerInfo *root,
+								RelOptInfo *baserel,
+								Oid foreigntableid);
+static ForeignScan *testGetForeignPlan(PlannerInfo *root,
+									   RelOptInfo *foreignrel,
+									   Oid foreigntableid,
+									   ForeignPath *best_path,
+									   List *tlist,
+									   List *scan_clauses,
+									   Plan *outer_plan);
+static void testBeginForeignScan(ForeignScanState *node, int eflags);
+static TupleTableSlot *testIterateForeignScan(ForeignScanState *node);
+static void testReScanForeignScan(ForeignScanState *node);
+static void testEndForeignScan(ForeignScanState *node);
+static void testBeginForeignModify(ModifyTableState *mtstate,
+								   ResultRelInfo *resultRelInfo,
+								   List *fdw_private,
+								   int subplan_index,
+								   int eflags);
+static TupleTableSlot *testExecForeignInsert(EState *estate,
+											 ResultRelInfo *resultRelInfo,
+											 TupleTableSlot *slot,
+											 TupleTableSlot *planSlot);
+static void testEndForeignModify(EState *estate,
+								 ResultRelInfo *resultRelInfo);
+static void testBeginForeignInsert(ModifyTableState *mtstate,
+								   ResultRelInfo *resultRelInfo);
+static void testEndForeignInsert(EState *estate,
+								 ResultRelInfo *resultRelInfo);
+static int	testIsForeignRelUpdatable(Relation rel);
+static void testPrepareForeignTransaction(FdwXactRslvState *state);
+static void testCommitForeignTransaction(FdwXactRslvState *state);
+static void testRollbackForeignTransaction(FdwXactRslvState *state);
+static char *testGetPrepareId(TransactionId xid, Oid serverid,
+							  Oid userid, int *prep_id_len);
+
+void
+_PG_init(void)
+{
+	DefineCustomBoolVariable("test_fdwxact.log_api_calls",
+							 "Report transaction API calls to logs.",
+							 NULL,
+							 &log_api_calls,
+							 false,
+							 PGC_USERSET,
+							 0,
+							 NULL, NULL, NULL);
+
+	RequestAddinShmemSpace(MAXALIGN(sizeof(testFdwXactSharedState)));
+	RequestNamedLWLockTranche("test_fdwxact", 1);
+
+	prev_shmem_startup_hook = shmem_startup_hook;
+	shmem_startup_hook = test_fdwxact_shmem_startup;
+}
+
+void
+_PG_fini(void)
+{
+	/* Uninstall hooks. */
+	shmem_startup_hook = prev_shmem_startup_hook;
+}
+
+static void
+test_fdwxact_shmem_startup(void)
+{
+	bool found;
+
+	if (prev_shmem_startup_hook)
+		prev_shmem_startup_hook();
+
+	fxss = ShmemInitStruct("test_fdwxact",
+						   sizeof(testFdwXactSharedState),
+						   &found);
+	if (!found)
+	{
+		memset(fxss->elevel, 0, TEST_FDWXCT_MAX_NAME_LEN);
+		memset(fxss->phase, 0, TEST_FDWXCT_MAX_NAME_LEN);
+		memset(fxss->server, 0, TEST_FDWXCT_MAX_NAME_LEN);
+		fxss->lock = &(GetNamedLWLockTranche("test_fdwxact"))->lock;
+	}
+}
+
+Datum
+test_fdw_handler(PG_FUNCTION_ARGS)
+{
+	FdwRoutine *routine = makeNode(FdwRoutine);
+
+	/* Functions for scanning foreign tables */
+	routine->GetForeignRelSize = testGetForeignRelSize;
+	routine->GetForeignPaths = testGetForeignPaths;
+	routine->GetForeignPlan = testGetForeignPlan;
+	routine->BeginForeignScan = testBeginForeignScan;
+	routine->IterateForeignScan = testIterateForeignScan;
+	routine->ReScanForeignScan = testReScanForeignScan;
+	routine->EndForeignScan = testEndForeignScan;
+
+	/* Functions for updating foreign tables */
+	routine->AddForeignUpdateTargets = NULL;
+	routine->PlanForeignModify = NULL;
+	routine->BeginForeignModify = testBeginForeignModify;
+	routine->ExecForeignInsert = testExecForeignInsert;
+	routine->EndForeignModify = testEndForeignModify;
+	routine->BeginForeignInsert = testBeginForeignInsert;
+	routine->EndForeignInsert = testEndForeignInsert;
+	routine->IsForeignRelUpdatable = testIsForeignRelUpdatable;
+
+	PG_RETURN_POINTER(routine);
+}
+
+Datum
+test_no2pc_fdw_handler(PG_FUNCTION_ARGS)
+{
+	FdwRoutine *routine = makeNode(FdwRoutine);
+
+	/* Functions for scanning foreign tables */
+	routine->GetForeignRelSize = testGetForeignRelSize;
+	routine->GetForeignPaths = testGetForeignPaths;
+	routine->GetForeignPlan = testGetForeignPlan;
+	routine->BeginForeignScan = testBeginForeignScan;
+	routine->IterateForeignScan = testIterateForeignScan;
+	routine->ReScanForeignScan = testReScanForeignScan;
+	routine->EndForeignScan = testEndForeignScan;
+
+	/* Functions for updating foreign tables */
+	routine->AddForeignUpdateTargets = NULL;
+	routine->PlanForeignModify = NULL;
+	routine->BeginForeignModify = testBeginForeignModify;
+	routine->ExecForeignInsert = testExecForeignInsert;
+	routine->EndForeignModify = testEndForeignModify;
+	routine->BeginForeignInsert = testBeginForeignInsert;
+	routine->EndForeignInsert = testEndForeignInsert;
+	routine->IsForeignRelUpdatable = testIsForeignRelUpdatable;
+
+	/* Support only COMMIT and ROLLBACK */
+	routine->CommitForeignTransaction = testCommitForeignTransaction;
+	routine->RollbackForeignTransaction = testRollbackForeignTransaction;
+
+	PG_RETURN_POINTER(routine);
+}
+
+Datum
+test_2pc_fdw_handler(PG_FUNCTION_ARGS)
+{
+	FdwRoutine *routine = makeNode(FdwRoutine);
+
+	/* Functions for scanning foreign tables */
+	routine->GetForeignRelSize = testGetForeignRelSize;
+	routine->GetForeignPaths = testGetForeignPaths;
+	routine->GetForeignPlan = testGetForeignPlan;
+	routine->BeginForeignScan = testBeginForeignScan;
+	routine->IterateForeignScan = testIterateForeignScan;
+	routine->ReScanForeignScan = testReScanForeignScan;
+	routine->EndForeignScan = testEndForeignScan;
+
+	/* Functions for updating foreign tables */
+	routine->AddForeignUpdateTargets = NULL;
+	routine->PlanForeignModify = NULL;
+	routine->BeginForeignModify = testBeginForeignModify;
+	routine->ExecForeignInsert = testExecForeignInsert;
+	routine->EndForeignModify = testEndForeignModify;
+	routine->BeginForeignInsert = testBeginForeignInsert;
+	routine->EndForeignInsert = testEndForeignInsert;
+	routine->IsForeignRelUpdatable = testIsForeignRelUpdatable;
+
+	/* Support all functions for foreign transactions */
+	routine->GetPrepareId = testGetPrepareId;
+	routine->PrepareForeignTransaction = testPrepareForeignTransaction;
+	routine->CommitForeignTransaction = testCommitForeignTransaction;
+	routine->RollbackForeignTransaction = testRollbackForeignTransaction;
+
+	PG_RETURN_POINTER(routine);
+}
+
+static void
+testGetForeignRelSize(PlannerInfo *root,
+					  RelOptInfo *baserel,
+					  Oid foreigntableid)
+{
+	baserel->pages = 10;
+	baserel->tuples = 100;
+}
+
+static void
+testGetForeignPaths(PlannerInfo *root,
+					RelOptInfo *baserel,
+					Oid foreigntableid)
+{
+	add_path(baserel, (Path *) create_foreignscan_path(root, baserel,
+													   NULL,
+													   10, 10, 10,
+													   NIL,
+													   baserel->lateral_relids,
+													   NULL, NIL));
+}
+
+static ForeignScan *
+testGetForeignPlan(PlannerInfo *root,
+				   RelOptInfo *foreignrel,
+				   Oid foreigntableid,
+				   ForeignPath *best_path,
+				   List *tlist,
+				   List *scan_clauses,
+				   Plan *outer_plan)
+{
+	return make_foreignscan(tlist,
+							NIL,
+							foreignrel->relid,
+							NIL,
+							NULL,
+							NIL,
+							NIL,
+							outer_plan);
+}
+
+static void
+testBeginForeignScan(ForeignScanState *node, int eflags)
+{
+	return;
+}
+
+static TupleTableSlot *
+testIterateForeignScan(ForeignScanState *node)
+{
+	return ExecClearTuple(node->ss.ss_ScanTupleSlot);
+}
+
+static void
+testReScanForeignScan(ForeignScanState *node)
+{
+	return;
+}
+
+static void
+testEndForeignScan(ForeignScanState *node)
+{
+	return;
+}
+
+static void
+testBeginForeignModify(ModifyTableState *mtstate,
+					   ResultRelInfo *resultRelInfo,
+					   List *fdw_private,
+					   int subplan_index,
+					   int eflags)
+{
+	return;
+}
+
+static TupleTableSlot *
+testExecForeignInsert(EState *estate,
+					  ResultRelInfo *resultRelInfo,
+					  TupleTableSlot *slot,
+					  TupleTableSlot *planSlot)
+{
+	return slot;
+}
+
+static void
+testEndForeignModify(EState *estate,
+					 ResultRelInfo *resultRelInfo)
+{
+	return;
+}
+
+static void
+testBeginForeignInsert(ModifyTableState *mtstate,
+					   ResultRelInfo *resultRelInfo)
+{
+	return;
+}
+
+static void
+testEndForeignInsert(EState *estate,
+					 ResultRelInfo *resultRelInfo)
+{
+	return;
+}
+
+static int
+testIsForeignRelUpdatable(Relation rel)
+{
+	/* allow only inserts */
+	return (1 << CMD_INSERT);
+}
+
+static char *
+testGetPrepareId(TransactionId xid, Oid serverid,
+				 Oid userid, int *prep_id_len)
+{
+	static char buf[32] = {0};
+
+	*prep_id_len = snprintf(buf, 32, "tx_%u", xid);
+
+	return buf;
+}
+
+static void
+testPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	int elevel;
+
+	if (check_event(state->server->servername, "prepare", &elevel))
+		elog(elevel, "injected error at prepare");
+
+	if (log_api_calls)
+		ereport(LOG, (errmsg("prepare %s on %s",
+							 state->fdwxact_id,
+							 state->server->servername)));
+}
+
+static void
+testCommitForeignTransaction(FdwXactRslvState *state)
+{
+	int elevel;
+
+	if (check_event(state->server->servername, "commit", &elevel))
+		elog(elevel, "injected error at commit");
+
+	if (log_api_calls)
+	{
+		if (state->flags && FDWXACT_FLAG_ONEPHASE)
+			ereport(LOG, (errmsg("commit %u on %s",
+								 state->xid, state->server->servername)));
+		else
+			ereport(LOG, (errmsg("commit prepared %s on %s",
+								 state->fdwxact_id,
+								 state->server->servername)));
+	}
+}
+
+static void
+testRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	if (log_api_calls)
+	{
+		if (state->flags && FDWXACT_FLAG_ONEPHASE)
+			ereport(LOG, (errmsg("rollback %u on %s",
+								 state->xid, state->server->servername)));
+		else
+			ereport(LOG, (errmsg("rollback prepared %s on %s",
+								 state->fdwxact_id,
+								 state->server->servername)));
+	}
+}
+
+/*
+ * Check if an event is set at the phase on the server. If there is, set
+ * elevel and return true.
+ */
+static bool
+check_event(char *servername, char *phase, int *elevel)
+{
+	LWLockAcquire(fxss->lock, LW_SHARED);
+
+	if (pg_strcasecmp(fxss->server, servername) != 0 ||
+		pg_strcasecmp(fxss->phase, phase) != 0)
+	{
+		LWLockRelease(fxss->lock);
+		return false;
+	}
+
+	/* Currently support only error and panic */
+	if (pg_strcasecmp(fxss->elevel, "error") == 0)
+		*elevel = ERROR;
+	if (pg_strcasecmp(fxss->elevel, "panic") == 0)
+		*elevel = PANIC;
+
+	LWLockRelease(fxss->lock);
+
+	return true;
+}
+
+/* SQL function to inject an error */
+Datum
+test_inject_error(PG_FUNCTION_ARGS)
+{
+	char *elevel = text_to_cstring(PG_GETARG_TEXT_P(0));
+	char *phase = text_to_cstring(PG_GETARG_TEXT_P(1));
+	char *server = text_to_cstring(PG_GETARG_TEXT_P(2));
+
+	LWLockAcquire(fxss->lock, LW_EXCLUSIVE);
+	strncpy(fxss->elevel, elevel, TEST_FDWXCT_MAX_NAME_LEN);
+	strncpy(fxss->phase, phase, TEST_FDWXCT_MAX_NAME_LEN);
+	strncpy(fxss->server, server, TEST_FDWXCT_MAX_NAME_LEN);
+	LWLockRelease(fxss->lock);
+
+	PG_RETURN_NULL();
+}
+
+/* SQL function to reset an error */
+Datum
+test_reset_error(PG_FUNCTION_ARGS)
+{
+	LWLockAcquire(fxss->lock, LW_EXCLUSIVE);
+	memset(fxss->elevel, 0, TEST_FDWXCT_MAX_NAME_LEN);
+	memset(fxss->phase, 0, TEST_FDWXCT_MAX_NAME_LEN);
+	memset(fxss->server, 0, TEST_FDWXCT_MAX_NAME_LEN);
+	LWLockRelease(fxss->lock);
+
+	PG_RETURN_NULL();
+}
diff --git a/src/test/modules/test_fdwxact/test_fdwxact.control b/src/test/modules/test_fdwxact/test_fdwxact.control
new file mode 100644
index 0000000000..ac9945ba03
--- /dev/null
+++ b/src/test/modules/test_fdwxact/test_fdwxact.control
@@ -0,0 +1,4 @@
+comment = 'Test code for fdwxact'
+default_version = '1.0'
+module_pathname = '$libdir/test_fdwxact'
+relocatable = true
diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile
index fa8e031526..d47d96975b 100644
--- a/src/test/recovery/Makefile
+++ b/src/test/recovery/Makefile
@@ -9,7 +9,7 @@
 #
 #-------------------------------------------------------------------------
 
-EXTRA_INSTALL=contrib/test_decoding
+EXTRA_INSTALL=contrib/test_decoding contrib/pageinspect contrib/postgres_fdw
 
 subdir = src/test/recovery
 top_builddir = ../../..
diff --git a/src/test/recovery/t/021_fdwxact.pl b/src/test/recovery/t/021_fdwxact.pl
new file mode 100644
index 0000000000..9af9bb81dc
--- /dev/null
+++ b/src/test/recovery/t/021_fdwxact.pl
@@ -0,0 +1,175 @@
+# Tests for transaction involving foreign servers
+use strict;
+use warnings;
+use PostgresNode;
+use TestLib;
+use Test::More tests => 7;
+
+# Setup master node
+my $node_master = get_new_node("master");
+my $node_standby = get_new_node("standby");
+
+$node_master->init(allows_streaming => 1);
+$node_master->append_conf('postgresql.conf', qq(
+max_prepared_transactions = 10
+max_prepared_foreign_transactions = 10
+max_foreign_transaction_resolvers = 2
+foreign_transaction_resolver_timeout = 0
+foreign_transaction_resolution_retry_interval = 5s
+foreign_twophase_commit = on
+));
+$node_master->start;
+
+# Take backup from master node
+my $backup_name = 'master_backup';
+$node_master->backup($backup_name);
+
+# Set up standby node
+$node_standby->init_from_backup($node_master, $backup_name,
+							   has_streaming => 1);
+$node_standby->start;
+
+# Set up foreign nodes
+my $node_fs1 = get_new_node("fs1");
+my $node_fs2 = get_new_node("fs2");
+my $fs1_port = $node_fs1->port;
+my $fs2_port = $node_fs2->port;
+$node_fs1->init;
+$node_fs2->init;
+$node_fs1->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs2->append_conf('postgresql.conf', qq(max_prepared_transactions = 10));
+$node_fs1->start;
+$node_fs2->start;
+
+# Create foreign servers on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE EXTENSION postgres_fdw
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs1_port');
+));
+$node_master->safe_psql('postgres', qq(
+CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw
+OPTIONS (dbname 'postgres', port '$fs2_port');
+));
+
+# Create user mapping on the master node
+$node_master->safe_psql('postgres', qq(
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs1;
+CREATE USER MAPPING FOR CURRENT_USER SERVER fs2;
+));
+
+# Create tables on foreign nodes and import them to the master node
+$node_fs1->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t1 (c int);
+));
+$node_fs2->safe_psql('postgres', qq(
+CREATE SCHEMA fs;
+CREATE TABLE fs.t2 (c int);
+));
+$node_master->safe_psql('postgres', qq(
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public;
+IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public;
+CREATE TABLE l_table (c int);
+));
+
+# Switch to synchronous replication
+$node_master->safe_psql('postgres', qq(
+ALTER SYSTEM SET synchronous_standby_names ='*';
+));
+$node_master->reload;
+
+my $result;
+
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node. Check if we can commit and rollback the foreign transactions
+# after the normal recovery.
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t2 VALUES (1);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (2);
+INSERT INTO t2 VALUES (2);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->stop;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after recovery');
+
+#
+# Prepare two transactions involving multiple foreign servers and shutdown
+# the master node immediately. Check if we can commit and rollback the foreign
+# transactions after the crash recovery.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (3);
+INSERT INTO t2 VALUES (3);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (4);
+INSERT INTO t2 VALUES (4);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+# Commit and rollback foreign transactions after the crash recovery.
+$result = $node_master->psql('postgres', qq(COMMIT PREPARED 'gxid1'));
+is($result, 0, 'Commit foreign transactions after crash recovery');
+$result = $node_master->psql('postgres', qq(ROLLBACK PREPARED 'gxid2'));
+is($result, 0, 'Rollback foreign transactions after crash recovery');
+
+#
+# Commit transaction involving foreign servers and shutdown the master node
+# immediately before checkpoint. Check that WAL replay cleans up
+# its shared memory state release locks while replaying transaction commit.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (5);
+INSERT INTO t2 VALUES (5);
+COMMIT;
+));
+
+$node_master->teardown_node;
+$node_master->start;
+
+$result = $node_master->safe_psql('postgres', qq(
+SELECT count(*) FROM pg_foreign_xacts;
+));
+is($result, 0, "Cleanup of shared memory state for foreign transactions");
+
+#
+# Check if the standby node can process prepared foreign transaction
+# after promotion.
+#
+$node_master->safe_psql('postgres', qq(
+BEGIN;
+INSERT INTO t1 VALUES (6);
+INSERT INTO t2 VALUES (6);
+PREPARE TRANSACTION 'gxid1';
+BEGIN;
+INSERT INTO t1 VALUES (7);
+INSERT INTO t2 VALUES (7);
+PREPARE TRANSACTION 'gxid2';
+));
+
+$node_master->teardown_node;
+$node_standby->promote;
+
+$result = $node_standby->psql('postgres', qq(COMMIT PREPARED 'gxid1';));
+is($result, 0, 'Commit foreign transaction after promotion');
+$result = $node_standby->psql('postgres', qq(ROLLBACK PREPARED 'gxid2';));
+is($result, 0, 'Rollback foreign transaction after promotion');
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index 38b2b1e8e1..f30fe6b492 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2335,9 +2335,12 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		 * Adjust the default postgresql.conf for regression testing. The user
 		 * can specify a file to be appended; in any case we expand logging
 		 * and set max_prepared_transactions to enable testing of prepared
-		 * xacts.  (Note: to reduce the probability of unexpected shmmax
-		 * failures, don't set max_prepared_transactions any higher than
-		 * actually needed by the prepared_xacts regression test.)
+		 * xacts.  We also set max_prepared_foreign_transactions and
+		 * max_foreign_transaction_resolvers to enable testing of transaction
+		 * involving multiple foreign servers. (Note: to reduce the probability
+		 * of unexpected shmmax failures, don't set max_prepared_transactions
+		 * any higher than actually needed by the prepared_xacts regression
+		 * test.)
 		 */
 		snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance);
 		pg_conf = fopen(buf, "a");
@@ -2352,7 +2355,9 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc
 		fputs("log_line_prefix = '%m %b[%p] %q%a '\n", pg_conf);
 		fputs("log_lock_waits = on\n", pg_conf);
 		fputs("log_temp_files = 128kB\n", pg_conf);
-		fputs("max_prepared_transactions = 2\n", pg_conf);
+		fputs("max_prepared_transactions = 3\n", pg_conf);
+		fputs("max_prepared_foreign_transactions = 2\n", pg_conf);
+		fputs("max_foreign_transaction_resolvers = 2\n", pg_conf);
 
 		for (sl = temp_configs; sl != NULL; sl = sl->next)
 		{
-- 
2.23.0

From b4d8e51c4731d388bdb63ebb3efbc6c275c696ca Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 21:28:58 +0500
Subject: [PATCH v22 4/6] postgres_fdw supports atomic commit APIs.

---
 contrib/postgres_fdw/Makefile                 |   7 +-
 contrib/postgres_fdw/connection.c             | 603 +++++++++++-------
 .../postgres_fdw/expected/postgres_fdw.out    | 280 +++++++-
 contrib/postgres_fdw/fdwxact.conf             |   3 +
 contrib/postgres_fdw/postgres_fdw.c           |  21 +-
 contrib/postgres_fdw/postgres_fdw.h           |   7 +-
 contrib/postgres_fdw/sql/postgres_fdw.sql     | 124 +++-
 doc/src/sgml/postgres-fdw.sgml                |  45 ++
 8 files changed, 831 insertions(+), 259 deletions(-)
 create mode 100644 contrib/postgres_fdw/fdwxact.conf

diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile
index ee8a80a392..91fa6e39fc 100644
--- a/contrib/postgres_fdw/Makefile
+++ b/contrib/postgres_fdw/Makefile
@@ -16,7 +16,7 @@ SHLIB_LINK_INTERNAL = $(libpq)
 EXTENSION = postgres_fdw
 DATA = postgres_fdw--1.0.sql
 
-REGRESS = postgres_fdw
+REGRESSCHECK = postgres_fdw
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -29,3 +29,8 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+check:
+	$(pg_regress_check) \
+	    --temp-config $(top_srcdir)/contrib/postgres_fdw/fdwxact.conf \
+	    $(REGRESSCHECK)
diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c
index 52d1fe3563..d55884b49b 100644
--- a/contrib/postgres_fdw/connection.c
+++ b/contrib/postgres_fdw/connection.c
@@ -1,7 +1,7 @@
 /*-------------------------------------------------------------------------
  *
  * connection.c
- *		  Connection management functions for postgres_fdw
+ *		  Connection and transaction management functions for postgres_fdw
  *
  * Portions Copyright (c) 2012-2020, PostgreSQL Global Development Group
  *
@@ -12,6 +12,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/xact.h"
 #include "catalog/pg_user_mapping.h"
@@ -56,6 +57,7 @@ typedef struct ConnCacheEntry
 	bool		have_error;		/* have any subxacts aborted in this xact? */
 	bool		changing_xact_state;	/* xact state change in process */
 	bool		invalidated;	/* true if reconnect is pending */
+	bool		xact_got_connection;
 	uint32		server_hashvalue;	/* hash value of foreign server OID */
 	uint32		mapping_hashvalue;	/* hash value of user mapping OID */
 } ConnCacheEntry;
@@ -69,17 +71,13 @@ static HTAB *ConnectionHash = NULL;
 static unsigned int cursor_number = 0;
 static unsigned int prep_stmt_number = 0;
 
-/* tracks whether any work is needed in callback functions */
-static bool xact_got_connection = false;
-
 /* prototypes of private functions */
 static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user);
 static void disconnect_pg_server(ConnCacheEntry *entry);
 static void check_conn_params(const char **keywords, const char **values, UserMapping *user);
 static void configure_remote_session(PGconn *conn);
 static void do_sql_command(PGconn *conn, const char *sql);
-static void begin_remote_xact(ConnCacheEntry *entry);
-static void pgfdw_xact_callback(XactEvent event, void *arg);
+static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid);
 static void pgfdw_subxact_callback(SubXactEvent event,
 								   SubTransactionId mySubid,
 								   SubTransactionId parentSubid,
@@ -92,23 +90,26 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query,
 static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime,
 									 PGresult **result);
 static bool UserMappingPasswordRequired(UserMapping *user);
+static void pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id,
+									bool is_commit);
+static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry);
+static ConnCacheEntry *GetConnectionState(Oid umid, bool will_prep_stmt,
+										  bool start_transaction);
+static ConnCacheEntry *GetConnectionCacheEntry(Oid umid);
 
 /*
- * Get a PGconn which can be used to execute queries on the remote PostgreSQL
- * server with the user's authorization.  A new connection is established
- * if we don't already have a suitable one, and a transaction is opened at
- * the right subtransaction nesting depth if we didn't do that already.
- *
- * will_prep_stmt must be true if caller intends to create any prepared
- * statements.  Since those don't go away automatically at transaction end
- * (not even on error), we need this flag to cue manual cleanup.
+ * Get connection cache entry. Unlike GetConenctionState function, this function
+ * doesn't establish new connection even if not yet.
  */
-PGconn *
-GetConnection(UserMapping *user, bool will_prep_stmt)
+static ConnCacheEntry *
+GetConnectionCacheEntry(Oid umid)
 {
-	bool		found;
 	ConnCacheEntry *entry;
-	ConnCacheKey key;
+	ConnCacheKey	key;
+	bool			found;
+
+	/* Create hash key for the entry.  Assume no pad bytes in key struct */
+	key = umid;
 
 	/* First time through, initialize connection cache hashtable */
 	if (ConnectionHash == NULL)
@@ -128,7 +129,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		 * Register some callback functions that manage connection cleanup.
 		 * This should be done just once in each backend.
 		 */
-		RegisterXactCallback(pgfdw_xact_callback, NULL);
 		RegisterSubXactCallback(pgfdw_subxact_callback, NULL);
 		CacheRegisterSyscacheCallback(FOREIGNSERVEROID,
 									  pgfdw_inval_callback, (Datum) 0);
@@ -136,12 +136,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 									  pgfdw_inval_callback, (Datum) 0);
 	}
 
-	/* Set flag that we did GetConnection during the current transaction */
-	xact_got_connection = true;
-
-	/* Create hash key for the entry.  Assume no pad bytes in key struct */
-	key = user->umid;
-
 	/*
 	 * Find or create cached entry for requested connection.
 	 */
@@ -155,6 +149,21 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->conn = NULL;
 	}
 
+	return entry;
+}
+
+/*
+ * This function gets the connection cache entry and establishes connection
+ * to the foreign server if there is no connection and starts a new transaction
+ * if 'start_transaction' is true.
+ */
+static ConnCacheEntry *
+GetConnectionState(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionCacheEntry(umid);
+
 	/* Reject further use of connections which failed abort cleanup. */
 	pgfdw_reject_incomplete_xact_state_change(entry);
 
@@ -182,6 +191,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	 */
 	if (entry->conn == NULL)
 	{
+		UserMapping	*user = GetUserMappingByOid(umid);
 		ForeignServer *server = GetForeignServer(user->serverid);
 
 		/* Reset all transient state fields, to be sure all are clean */
@@ -190,6 +200,7 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		entry->have_error = false;
 		entry->changing_xact_state = false;
 		entry->invalidated = false;
+		entry->xact_got_connection = false;
 		entry->server_hashvalue =
 			GetSysCacheHashValue1(FOREIGNSERVEROID,
 								  ObjectIdGetDatum(server->serverid));
@@ -200,6 +211,15 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 		/* Now try to make the connection */
 		entry->conn = connect_pg_server(server, user);
 
+		Assert(entry->conn);
+
+		if (!entry->conn)
+		{
+			elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed",
+				 server->servername);
+			return NULL;
+		}
+
 		elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)",
 			 entry->conn, server->servername, user->umid, user->userid);
 	}
@@ -207,11 +227,39 @@ GetConnection(UserMapping *user, bool will_prep_stmt)
 	/*
 	 * Start a new transaction or subtransaction if needed.
 	 */
-	begin_remote_xact(entry);
+	if (start_transaction)
+	{
+		UserMapping		*user = GetUserMappingByOid(umid);
+
+		begin_remote_xact(entry, user->serverid, user->userid);
+
+		/* Set flag that we did GetConnection during the current transaction */
+		entry->xact_got_connection = true;
+	}
 
 	/* Remember if caller will prepare statements */
 	entry->have_prep_stmt |= will_prep_stmt;
 
+	return entry;
+}
+
+/*
+ * Get a PGconn which can be used to execute queries on the remote PostgreSQL
+ * server with the user's authorization.  A new connection is established
+ * if we don't already have a suitable one, and a transaction is opened at
+ * the right subtransaction nesting depth if we didn't do that already.
+ *
+ * will_prep_stmt must be true if caller intends to create any prepared
+ * statements.  Since those don't go away automatically at transaction end
+ * (not even on error), we need this flag to cue manual cleanup.
+ */
+PGconn *
+GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction)
+{
+	ConnCacheEntry *entry;
+
+	entry = GetConnectionState(umid, will_prep_stmt, start_transaction);
+
 	return entry->conn;
 }
 
@@ -473,7 +521,7 @@ do_sql_command(PGconn *conn, const char *sql)
  * control which remote queries share a snapshot.
  */
 static void
-begin_remote_xact(ConnCacheEntry *entry)
+begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid)
 {
 	int			curlevel = GetCurrentTransactionNestLevel();
 
@@ -700,193 +748,6 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 	PG_END_TRY();
 }
 
-/*
- * pgfdw_xact_callback --- cleanup at main-transaction end.
- */
-static void
-pgfdw_xact_callback(XactEvent event, void *arg)
-{
-	HASH_SEQ_STATUS scan;
-	ConnCacheEntry *entry;
-
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
-	/*
-	 * Scan all connection cache entries to find open remote transactions, and
-	 * close them.
-	 */
-	hash_seq_init(&scan, ConnectionHash);
-	while ((entry = (ConnCacheEntry *) hash_seq_search(&scan)))
-	{
-		PGresult   *res;
-
-		/* Ignore cache entry if no open connection right now */
-		if (entry->conn == NULL)
-			continue;
-
-		/* If it has an open remote transaction, try to close it */
-		if (entry->xact_depth > 0)
-		{
-			bool		abort_cleanup_failure = false;
-
-			elog(DEBUG3, "closing remote transaction on connection %p",
-				 entry->conn);
-
-			switch (event)
-			{
-				case XACT_EVENT_PARALLEL_PRE_COMMIT:
-				case XACT_EVENT_PRE_COMMIT:
-
-					/*
-					 * If abort cleanup previously failed for this connection,
-					 * we can't issue any more commands against it.
-					 */
-					pgfdw_reject_incomplete_xact_state_change(entry);
-
-					/* Commit all remote transactions during pre-commit */
-					entry->changing_xact_state = true;
-					do_sql_command(entry->conn, "COMMIT TRANSACTION");
-					entry->changing_xact_state = false;
-
-					/*
-					 * If there were any errors in subtransactions, and we
-					 * made prepared statements, do a DEALLOCATE ALL to make
-					 * sure we get rid of all prepared statements. This is
-					 * annoying and not terribly bulletproof, but it's
-					 * probably not worth trying harder.
-					 *
-					 * DEALLOCATE ALL only exists in 8.3 and later, so this
-					 * constrains how old a server postgres_fdw can
-					 * communicate with.  We intentionally ignore errors in
-					 * the DEALLOCATE, so that we can hobble along to some
-					 * extent with older servers (leaking prepared statements
-					 * as we go; but we don't really support update operations
-					 * pre-8.3 anyway).
-					 */
-					if (entry->have_prep_stmt && entry->have_error)
-					{
-						res = PQexec(entry->conn, "DEALLOCATE ALL");
-						PQclear(res);
-					}
-					entry->have_prep_stmt = false;
-					entry->have_error = false;
-					break;
-				case XACT_EVENT_PRE_PREPARE:
-
-					/*
-					 * We disallow any remote transactions, since it's not
-					 * very reasonable to hold them open until the prepared
-					 * transaction is committed.  For the moment, throw error
-					 * unconditionally; later we might allow read-only cases.
-					 * Note that the error will cause us to come right back
-					 * here with event == XACT_EVENT_ABORT, so we'll clean up
-					 * the connection state at that point.
-					 */
-					ereport(ERROR,
-							(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-							 errmsg("cannot PREPARE a transaction that has operated on postgres_fdw foreign tables")));
-					break;
-				case XACT_EVENT_PARALLEL_COMMIT:
-				case XACT_EVENT_COMMIT:
-				case XACT_EVENT_PREPARE:
-					/* Pre-commit should have closed the open transaction */
-					elog(ERROR, "missed cleaning up connection during pre-commit");
-					break;
-				case XACT_EVENT_PARALLEL_ABORT:
-				case XACT_EVENT_ABORT:
-
-					/*
-					 * Don't try to clean up the connection if we're already
-					 * in error recursion trouble.
-					 */
-					if (in_error_recursion_trouble())
-						entry->changing_xact_state = true;
-
-					/*
-					 * If connection is already unsalvageable, don't touch it
-					 * further.
-					 */
-					if (entry->changing_xact_state)
-						break;
-
-					/*
-					 * Mark this connection as in the process of changing
-					 * transaction state.
-					 */
-					entry->changing_xact_state = true;
-
-					/* Assume we might have lost track of prepared statements */
-					entry->have_error = true;
-
-					/*
-					 * If a command has been submitted to the remote server by
-					 * using an asynchronous execution function, the command
-					 * might not have yet completed.  Check to see if a
-					 * command is still being processed by the remote server,
-					 * and if so, request cancellation of the command.
-					 */
-					if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
-						!pgfdw_cancel_query(entry->conn))
-					{
-						/* Unable to cancel running query. */
-						abort_cleanup_failure = true;
-					}
-					else if (!pgfdw_exec_cleanup_query(entry->conn,
-													   "ABORT TRANSACTION",
-													   false))
-					{
-						/* Unable to abort remote transaction. */
-						abort_cleanup_failure = true;
-					}
-					else if (entry->have_prep_stmt && entry->have_error &&
-							 !pgfdw_exec_cleanup_query(entry->conn,
-													   "DEALLOCATE ALL",
-													   true))
-					{
-						/* Trouble clearing prepared statements. */
-						abort_cleanup_failure = true;
-					}
-					else
-					{
-						entry->have_prep_stmt = false;
-						entry->have_error = false;
-					}
-
-					/* Disarm changing_xact_state if it all worked. */
-					entry->changing_xact_state = abort_cleanup_failure;
-					break;
-			}
-		}
-
-		/* Reset state to show we're out of a transaction */
-		entry->xact_depth = 0;
-
-		/*
-		 * If the connection isn't in a good idle state, discard it to
-		 * recover. Next GetConnection will open a new connection.
-		 */
-		if (PQstatus(entry->conn) != CONNECTION_OK ||
-			PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
-			entry->changing_xact_state)
-		{
-			elog(DEBUG3, "discarding connection %p", entry->conn);
-			disconnect_pg_server(entry);
-		}
-	}
-
-	/*
-	 * Regardless of the event type, we can now mark ourselves as out of the
-	 * transaction.  (Note: if we are here during PRE_COMMIT or PRE_PREPARE,
-	 * this saves a useless scan of the hashtable during COMMIT or PREPARE.)
-	 */
-	xact_got_connection = false;
-
-	/* Also reset cursor numbering for next transaction */
-	cursor_number = 0;
-}
-
 /*
  * pgfdw_subxact_callback --- cleanup at subtransaction end.
  */
@@ -903,10 +764,6 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 		  event == SUBXACT_EVENT_ABORT_SUB))
 		return;
 
-	/* Quick exit if no connections were touched in this transaction. */
-	if (!xact_got_connection)
-		return;
-
 	/*
 	 * Scan all connection cache entries to find open remote subtransactions
 	 * of the current level, and close them.
@@ -917,6 +774,10 @@ pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid,
 	{
 		char		sql[100];
 
+		/* Quick exit if no connections were touched in this transaction. */
+		if (!entry->xact_got_connection)
+			continue;
+
 		/*
 		 * We only care about connections with open remote subtransactions of
 		 * the current level.
@@ -1251,3 +1112,309 @@ exit:	;
 		*result = last_res;
 	return timed_out;
 }
+
+/*
+ * Prepare a transaction on foreign server.
+ */
+void
+postgresPrepareForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	PGresult	*res;
+	StringInfo	command;
+
+	/* The transaction should have started already get the cache entry */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+
+	/* The transaction should have been started */
+	Assert(entry->xact_got_connection && entry->conn);
+
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	command = makeStringInfo();
+	appendStringInfo(command, "PREPARE TRANSACTION '%s'", state->fdwxact_id);
+
+	/* Do commit foreign transaction */
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, command->data);
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not prepare transaction on server %s with ID %s",
+							   state->server->servername, state->fdwxact_id)));
+
+	elog(DEBUG1, "prepared foreign transaction on server %s with ID %s",
+		 state->server->servername, state->fdwxact_id);
+
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Commit a transaction or a prepared transaction on foreign server. If
+ * state->flags contains FDWXACT_FLAG_ONEPHASE this function can commit the
+ * foreign transaction without preparation, otherwise commit the prepared
+ * transaction.
+ */
+void
+postgresCommitForeignTransaction(FdwXactRslvState *state)
+{
+	ConnCacheEntry *entry = NULL;
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	PGresult		*res;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* COMMIT PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, true);
+		return;
+	}
+
+	/*
+	 * In simple commit case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	if (!entry->conn || !entry->xact_got_connection)
+		return;
+
+	/*
+	 * If abort cleanup previously failed for this connection, we can't issue
+	 * any more commands against it.
+	 */
+	pgfdw_reject_incomplete_xact_state_change(entry);
+
+	entry->changing_xact_state = true;
+	res = pgfdw_exec_query(entry->conn, "COMMIT TRANSACTION");
+	entry->changing_xact_state = false;
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+		ereport(ERROR, (errmsg("could not commit transaction on server %s",
+							   state->server->servername)));
+
+	/*
+	 * If there were any errors in subtransactions, and we ma
+	 * made prepared statements, do a DEALLOCATE ALL to make
+	 * sure we get rid of all prepared statements. This is
+	 * annoying and not terribly bulletproof, but it's
+	 * probably not worth trying harder.
+	 *
+	 * DEALLOCATE ALL only exists in 8.3 and later, so this
+	 * constrains how old a server postgres_fdw can
+	 * communicate with.  We intentionally ignore errors in
+	 * the DEALLOCATE, so that we can hobble along to some
+	 * extent with older servers (leaking prepared statements
+	 * as we go; but we don't really support update operations
+	 * pre-8.3 anyway).
+	 */
+	if (entry->have_prep_stmt && entry->have_error)
+	{
+		res = PQexec(entry->conn, "DEALLOCATE ALL");
+		PQclear(res);
+	}
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/*
+ * Rollback a transaction on foreign server. As with commit case, if state->flags
+ * contains FDWAXCT_FLAG_ONEPHASE this function can rollback the foreign
+ * transaction without preparation, other wise rollback the prepared transaction.
+ * This function must tolerate to being called recursively as an error can happen
+ * during aborting.
+ */
+void
+postgresRollbackForeignTransaction(FdwXactRslvState *state)
+{
+	bool			is_onephase = (state->flags & FDWXACT_FLAG_ONEPHASE) != 0;
+	ConnCacheEntry *entry = NULL;
+	bool abort_cleanup_failure = false;
+
+	if (!is_onephase)
+	{
+		/*
+		 * In two-phase commit case, the foreign transaction has prepared and
+		 * closed, so we might not have a connection to it. We get a connection
+		 * but don't start transaction.
+		 */
+		entry = GetConnectionState(state->usermapping->umid, false, false);
+
+		/* ROLLBACK PREPARED the transaction */
+		pgfdw_end_prepared_xact(entry, state->fdwxact_id, false);
+		return;
+	}
+
+	/*
+	 * In simple rollback case, we must have a connection to the foreign server
+	 * because the foreign transaction is not closed yet. We get the connection
+	 * entry from the cache.
+	 */
+	entry = GetConnectionCacheEntry(state->usermapping->umid);
+	Assert(entry);
+
+	/*
+	 * Cleanup connection entry transaction if transaction fails before
+	 * establishing a connection or starting transaction.
+	 */
+	if (!entry->conn || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Don't try to clean up the connection if we're already
+	 * in error recursion trouble.
+	 */
+	if (in_error_recursion_trouble())
+		entry->changing_xact_state = true;
+
+	/*
+	 * If connection is before starting transaction or is already unsalvageable,
+	 * do only the cleanup and don't touch it further.
+	 */
+	if (entry->changing_xact_state || !entry->xact_got_connection)
+	{
+		pgfdw_cleanup_after_transaction(entry);
+		return;
+	}
+
+	/*
+	 * Mark this connection as in the process of changing
+	 * transaction state.
+	 */
+	entry->changing_xact_state = true;
+
+	/* Assume we might have lost track of prepared statements */
+	entry->have_error = true;
+
+	/*
+	 * If a command has been submitted to the remote server by
+	 * using an asynchronous execution function, the command
+	 * might not have yet completed.  Check to see if a
+	 * command is still being processed by the remote server,
+	 * and if so, request cancellation of the command.
+	 */
+	if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE &&
+		!pgfdw_cancel_query(entry->conn))
+	{
+		/* Unable to cancel running query. */
+		abort_cleanup_failure = true;
+	}
+	else if (!pgfdw_exec_cleanup_query(entry->conn,
+									   "ABORT TRANSACTION",
+									   false))
+	{
+		/* Unable to abort remote transaction. */
+		abort_cleanup_failure = true;
+	}
+	else if (entry->have_prep_stmt && entry->have_error &&
+			 !pgfdw_exec_cleanup_query(entry->conn,
+									   "DEALLOCATE ALL",
+									   true))
+	{
+		/* Trouble clearing prepared statements. */
+		abort_cleanup_failure = true;
+	}
+
+	/* Disarm changing_xact_state if it all worked. */
+	entry->changing_xact_state = abort_cleanup_failure;
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+
+	return;
+}
+
+/*
+ * Commit or rollback prepared transaction on the foreign server.
+ */
+static void
+pgfdw_end_prepared_xact(ConnCacheEntry *entry, char *fdwxact_id, bool is_commit)
+{
+	StringInfo	command;
+	PGresult	*res;
+
+	command = makeStringInfo();
+	appendStringInfo(command, "%s PREPARED '%s'",
+					 is_commit ? "COMMIT" : "ROLLBACK",
+					 fdwxact_id);
+
+	res = pgfdw_exec_query(entry->conn, command->data);
+
+	if (PQresultStatus(res) != PGRES_COMMAND_OK)
+	{
+		int		sqlstate;
+		char	*diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
+
+		if (diag_sqlstate)
+		{
+			sqlstate = MAKE_SQLSTATE(diag_sqlstate[0],
+									 diag_sqlstate[1],
+									 diag_sqlstate[2],
+									 diag_sqlstate[3],
+									 diag_sqlstate[4]);
+		}
+		else
+			sqlstate = ERRCODE_CONNECTION_FAILURE;
+
+		/*
+		 * As core global transaction manager states, it's possible that the
+		 * given foreign transaction doesn't exist on the foreign server. So
+		 * we should accept an UNDEFINED_OBJECT error.
+		 */
+		if (sqlstate != ERRCODE_UNDEFINED_OBJECT)
+			pgfdw_report_error(ERROR, res, entry->conn, false, command->data);
+	}
+
+	elog(DEBUG1, "%s prepared foreign transaction with ID %s",
+		 is_commit ? "commit" : "rollback",
+		 fdwxact_id);
+
+	/* Cleanup transaction status */
+	pgfdw_cleanup_after_transaction(entry);
+}
+
+/* Cleanup at main-transaction end */
+static void
+pgfdw_cleanup_after_transaction(ConnCacheEntry *entry)
+{
+	/* Reset state to show we're out of a transaction */
+	entry->xact_depth = 0;
+	entry->have_prep_stmt = false;
+	entry->have_error  = false;
+	entry->xact_got_connection = false;
+
+	/*
+	 * If the connection isn't in a good idle state, discard it to
+	 * recover. Next GetConnection will open a new connection.
+	 */
+	if (PQstatus(entry->conn) != CONNECTION_OK ||
+		PQtransactionStatus(entry->conn) != PQTRANS_IDLE ||
+		entry->changing_xact_state)
+	{
+		elog(DEBUG3, "discarding connection %p", entry->conn);
+		disconnect_pg_server(entry);
+	}
+
+	entry->changing_xact_state = false;
+
+	/* Also reset cursor numbering for next transaction */
+	cursor_number = 0;
+}
diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 82fc1290ef..dbdd4cc32c 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -13,12 +13,17 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 -- ===================================================================
 -- create objects used through FDW loopback server
 -- ===================================================================
@@ -52,6 +57,13 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
 ALTER TABLE "S 1"."T 2" SET (autovacuum_enabled = 'false');
@@ -87,6 +99,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 -- ===================================================================
 -- create foreign tables
 -- ===================================================================
@@ -129,6 +142,12 @@ CREATE FOREIGN TABLE ft6 (
 	c2 int NOT NULL,
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -191,15 +210,17 @@ ALTER FOREIGN TABLE ft2 OPTIONS (schema_name 'S 1', table_name 'T 1');
 ALTER FOREIGN TABLE ft1 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1');
 \det+
-                              List of foreign tables
- Schema | Table |  Server   |              FDW options              | Description 
---------+-------+-----------+---------------------------------------+-------------
- public | ft1   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft2   | loopback  | (schema_name 'S 1', table_name 'T 1') | 
- public | ft4   | loopback  | (schema_name 'S 1', table_name 'T 3') | 
- public | ft5   | loopback  | (schema_name 'S 1', table_name 'T 4') | 
- public | ft6   | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
-(5 rows)
+                               List of foreign tables
+ Schema |  Table  |  Server   |              FDW options              | Description 
+--------+---------+-----------+---------------------------------------+-------------
+ public | ft1     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft2     | loopback  | (schema_name 'S 1', table_name 'T 1') | 
+ public | ft4     | loopback  | (schema_name 'S 1', table_name 'T 3') | 
+ public | ft5     | loopback  | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft6     | loopback2 | (schema_name 'S 1', table_name 'T 4') | 
+ public | ft7_2pc | loopback  | (schema_name 'S 1', table_name 'T 5') | 
+ public | ft8_2pc | loopback2 | (schema_name 'S 1', table_name 'T 5') | 
+(7 rows)
 
 -- Test that alteration of server options causes reconnection
 -- Remote's errors might be non-English, so hide them to ensure stable results
@@ -8923,10 +8944,10 @@ RESET ROLE;
 ALTER USER MAPPING FOR regress_nosuper SERVER loopback_nopw OPTIONS (ADD password_required 'false');
 SET ROLE regress_nosuper;
 -- Should finally work now
-SELECT * FROM ft1_nopw LIMIT 1;
-  c1  | c2 | c3 | c4 | c5 | c6 |     c7     | c8 
-------+----+----+----+----+----+------------+----
- 1111 |  2 |    |    |    |    | ft1        | 
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
+ c1 | c2 |        c3         |              c4              |            c5            | c6 |     c7     | c8  
+----+----+-------------------+------------------------------+--------------------------+----+------------+-----
+  1 |  2 | 00001_trig_update | Fri Jan 02 00:00:00 1970 PST | Fri Jan 02 00:00:00 1970 | 1  | 1          | foo
 (1 row)
 
 -- unpriv user also cannot set sslcert / sslkey on the user mapping
@@ -8943,16 +8964,16 @@ HINT:  User mappings with the sslcert or sslkey options set may only be created
 DROP USER MAPPING FOR CURRENT_USER SERVER loopback_nopw;
 -- This will fail again as it'll resolve the user mapping for public, which
 -- lacks password_required=false
-SELECT * FROM ft1_nopw LIMIT 1;
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
 ERROR:  password is required
 DETAIL:  Non-superusers must provide a password in the user mapping.
 RESET ROLE;
 -- The user mapping for public is passwordless and lacks the password_required=false
 -- mapping option, but will work because the current user is a superuser.
 SELECT * FROM ft1_nopw LIMIT 1;
-  c1  | c2 | c3 | c4 | c5 | c6 |     c7     | c8 
-------+----+----+----+----+----+------------+----
- 1111 |  2 |    |    |    |    | ft1        | 
+ c1 | c2 |  c3   |              c4              |            c5            | c6 |     c7     | c8  
+----+----+-------+------------------------------+--------------------------+----+------------+-----
+  6 |  6 | 00006 | Wed Jan 07 00:00:00 1970 PST | Wed Jan 07 00:00:00 1970 | 6  | 6          | foo
 (1 row)
 
 -- cleanup
@@ -8961,16 +8982,225 @@ DROP OWNED BY regress_nosuper;
 DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
--- Two-phase transactions are not supported.
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+-- Modify single foreign server and then commit and rollback.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+(1 row)
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+(3 rows)
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
 BEGIN;
-SELECT count(*) FROM ft1;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ERROR:  duplicate key value violates unique constraint "t6_pkey"
+DETAIL:  Key (c1)=(3) already exists.
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+SELECT * FROM "S 1"."T 6";
+ c1 
+----
+  3
+(1 row)
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ERROR:  null value in column "c1" of relation "T 5" violates not-null constraint
+DETAIL:  Failing row contains (null).
+CONTEXT:  remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1)
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+(4 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+(6 rows)
+
+RELEASE SAVEPOINT S1;
+ERROR:  RELEASE SAVEPOINT can only be used in transaction blocks
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+(8 rows)
+
+SET foreign_twophase_commit TO 'required';
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+ c1 
+----
+  1
+  2
+  2
+  3
+  5
+  5
+  8
+  8
+  9
+  9
+(10 rows)
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
  count 
 -------
-   822
+     0
 (1 row)
 
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
-ERROR:  cannot PREPARE a transaction that has operated on postgres_fdw foreign tables
-ROLLBACK;
-WARNING:  there is no transaction in progress
diff --git a/contrib/postgres_fdw/fdwxact.conf b/contrib/postgres_fdw/fdwxact.conf
new file mode 100644
index 0000000000..3fdbf93cdb
--- /dev/null
+++ b/contrib/postgres_fdw/fdwxact.conf
@@ -0,0 +1,3 @@
+max_prepared_transactions = 3
+max_prepared_foreign_transactions = 3
+max_foreign_transaction_resolvers = 2
diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c
index 9fc53cad68..105451d199 100644
--- a/contrib/postgres_fdw/postgres_fdw.c
+++ b/contrib/postgres_fdw/postgres_fdw.c
@@ -14,6 +14,7 @@
 
 #include <limits.h>
 
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
@@ -504,7 +505,6 @@ static void merge_fdw_options(PgFdwRelationInfo *fpinfo,
 							  const PgFdwRelationInfo *fpinfo_o,
 							  const PgFdwRelationInfo *fpinfo_i);
 
-
 /*
  * Foreign-data wrapper handler function: return a struct with pointers
  * to my callback routines.
@@ -558,6 +558,11 @@ postgres_fdw_handler(PG_FUNCTION_ARGS)
 	/* Support functions for upper relation push-down */
 	routine->GetForeignUpperPaths = postgresGetForeignUpperPaths;
 
+	/* Support functions for foreign transactions */
+	routine->PrepareForeignTransaction = postgresPrepareForeignTransaction;
+	routine->CommitForeignTransaction = postgresCommitForeignTransaction;
+	routine->RollbackForeignTransaction = postgresRollbackForeignTransaction;
+
 	PG_RETURN_POINTER(routine);
 }
 
@@ -1434,7 +1439,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	fsstate->conn = GetConnection(user, false);
+	fsstate->conn = GetConnection(user->umid, false, true);
 
 	/* Assign a unique ID for my cursor */
 	fsstate->cursor_number = GetCursorNumber(fsstate->conn);
@@ -2372,7 +2377,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags)
 	 * Get connection to the foreign server.  Connection manager will
 	 * establish new connection if necessary.
 	 */
-	dmstate->conn = GetConnection(user, false);
+	dmstate->conn = GetConnection(user->umid, false, true);
 
 	/* Update the foreign-join-related fields. */
 	if (fsplan->scan.scanrelid == 0)
@@ -2746,7 +2751,7 @@ estimate_path_cost_size(PlannerInfo *root,
 								false, &retrieved_attrs, NULL);
 
 		/* Get the remote estimate */
-		conn = GetConnection(fpinfo->user, false);
+		conn = GetConnection(fpinfo->user->umid, false, true);
 		get_remote_estimate(sql.data, conn, &rows, &width,
 							&startup_cost, &total_cost);
 		ReleaseConnection(conn);
@@ -3566,7 +3571,7 @@ create_foreign_modify(EState *estate,
 	user = GetUserMapping(userid, table->serverid);
 
 	/* Open connection; report that we'll create a prepared statement. */
-	fmstate->conn = GetConnection(user, true);
+	fmstate->conn = GetConnection(user->umid, true, true);
 	fmstate->p_name = NULL;		/* prepared statement not made yet */
 
 	/* Set up remote query information. */
@@ -4441,7 +4446,7 @@ postgresAnalyzeForeignTable(Relation relation,
 	 */
 	table = GetForeignTable(RelationGetRelid(relation));
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct command to get page count for relation.
@@ -4527,7 +4532,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel,
 	table = GetForeignTable(RelationGetRelid(relation));
 	server = GetForeignServer(table->serverid);
 	user = GetUserMapping(relation->rd_rel->relowner, table->serverid);
-	conn = GetConnection(user, false);
+	conn = GetConnection(user->umid, false, true);
 
 	/*
 	 * Construct cursor that retrieves whole rows from remote.
@@ -4755,7 +4760,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid)
 	 */
 	server = GetForeignServer(serverOid);
 	mapping = GetUserMapping(GetUserId(), server->serverid);
-	conn = GetConnection(mapping, false);
+	conn = GetConnection(mapping->umid, false, true);
 
 	/* Don't attempt to import collation if remote server hasn't got it */
 	if (PQserverVersion(conn) < 90100)
diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h
index eef410db39..43ffd4f73f 100644
--- a/contrib/postgres_fdw/postgres_fdw.h
+++ b/contrib/postgres_fdw/postgres_fdw.h
@@ -13,6 +13,7 @@
 #ifndef POSTGRES_FDW_H
 #define POSTGRES_FDW_H
 
+#include "access/fdwxact.h"
 #include "foreign/foreign.h"
 #include "lib/stringinfo.h"
 #include "libpq-fe.h"
@@ -129,7 +130,7 @@ extern int	set_transmission_modes(void);
 extern void reset_transmission_modes(int nestlevel);
 
 /* in connection.c */
-extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt);
+extern PGconn *GetConnection(Oid umid, bool will_prep_stmt, bool start_transaction);
 extern void ReleaseConnection(PGconn *conn);
 extern unsigned int GetCursorNumber(PGconn *conn);
 extern unsigned int GetPrepStmtNumber(PGconn *conn);
@@ -137,6 +138,9 @@ extern PGresult *pgfdw_get_result(PGconn *conn, const char *query);
 extern PGresult *pgfdw_exec_query(PGconn *conn, const char *query);
 extern void pgfdw_report_error(int elevel, PGresult *res, PGconn *conn,
 							   bool clear, const char *sql);
+extern void postgresPrepareForeignTransaction(FdwXactRslvState *state);
+extern void postgresCommitForeignTransaction(FdwXactRslvState *state);
+extern void postgresRollbackForeignTransaction(FdwXactRslvState *state);
 
 /* in option.c */
 extern int	ExtractConnectionOptions(List *defelems,
@@ -203,6 +207,7 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root,
 									bool is_subquery,
 									List **retrieved_attrs, List **params_list);
 extern const char *get_jointype_name(JoinType jointype);
+extern bool server_uses_twophase_commit(ForeignServer *server);
 
 /* in shippable.c */
 extern bool is_builtin(Oid objectId);
diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql
index 83971665e3..1ef66123df 100644
--- a/contrib/postgres_fdw/sql/postgres_fdw.sql
+++ b/contrib/postgres_fdw/sql/postgres_fdw.sql
@@ -15,6 +15,10 @@ DO $d$
             OPTIONS (dbname '$$||current_database()||$$',
                      port '$$||current_setting('port')||$$'
             )$$;
+        EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
     END;
 $d$;
 
@@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1
 	OPTIONS (user 'value', password 'value');
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback;
 CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2;
+CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3;
 
 -- ===================================================================
 -- create objects used through FDW loopback server
@@ -56,6 +61,14 @@ CREATE TABLE "S 1"."T 4" (
 	c3 text,
 	CONSTRAINT t4_pkey PRIMARY KEY (c1)
 );
+CREATE TABLE "S 1"."T 5" (
+       c1 int NOT NULL
+);
+
+CREATE TABLE "S 1"."T 6" (
+       c1 int NOT NULL,
+       CONSTRAINT t6_pkey PRIMARY KEY (c1)
+);
 
 -- Disable autovacuum for these tables to avoid unexpected effects of that
 ALTER TABLE "S 1"."T 1" SET (autovacuum_enabled = 'false');
@@ -94,6 +107,7 @@ ANALYZE "S 1"."T 1";
 ANALYZE "S 1"."T 2";
 ANALYZE "S 1"."T 3";
 ANALYZE "S 1"."T 4";
+ANALYZE "S 1"."T 5";
 
 -- ===================================================================
 -- create foreign tables
@@ -142,6 +156,15 @@ CREATE FOREIGN TABLE ft6 (
 	c3 text
 ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4');
 
+CREATE FOREIGN TABLE ft7_2pc (
+       c1 int NOT NULL
+) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+CREATE FOREIGN TABLE ft8_2pc (
+       c1 int NOT NULL
+) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5');
+
+
 -- ===================================================================
 -- tests for validator
 -- ===================================================================
@@ -2598,7 +2621,7 @@ ALTER USER MAPPING FOR regress_nosuper SERVER loopback_nopw OPTIONS (ADD passwor
 SET ROLE regress_nosuper;
 
 -- Should finally work now
-SELECT * FROM ft1_nopw LIMIT 1;
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
 
 -- unpriv user also cannot set sslcert / sslkey on the user mapping
 -- first set password_required so we see the right error messages
@@ -2612,7 +2635,7 @@ DROP USER MAPPING FOR CURRENT_USER SERVER loopback_nopw;
 
 -- This will fail again as it'll resolve the user mapping for public, which
 -- lacks password_required=false
-SELECT * FROM ft1_nopw LIMIT 1;
+SELECT * FROM ft1_nopw ORDER BY 1 LIMIT 1;
 
 RESET ROLE;
 
@@ -2628,9 +2651,98 @@ DROP ROLE regress_nosuper;
 -- Clean-up
 RESET enable_partitionwise_aggregate;
 
--- Two-phase transactions are not supported.
+-- ===================================================================
+-- test distributed atomic commit across foreign servers
+-- ===================================================================
+
+-- Enable atomic commit
+SET foreign_twophase_commit TO 'required';
+
+-- Modify single foreign server and then commit and rollback.
 BEGIN;
-SELECT count(*) FROM ft1;
--- error here
-PREPARE TRANSACTION 'fdw_tpc';
+INSERT INTO ft7_2pc VALUES(1);
+COMMIT;
+SELECT * FROM ft7_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(1);
 ROLLBACK;
+SELECT * FROM ft7_2pc;
+
+-- Modify two servers then commit and rollback. This requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+COMMIT;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(2);
+INSERT INTO ft8_2pc VALUES(2);
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+-- Modify both local data and 2PC-capable server then commit and rollback.
+-- This also requires to use 2PC.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(3);
+INSERT INTO "S 1"."T 6" VALUES (3);
+ROLLBACK;
+SELECT * FROM ft7_2pc;
+SELECT * FROM "S 1"."T 6";
+
+-- Modify foreign server and raise an error. No data changed.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(4);
+INSERT INTO ft8_2pc VALUES(NULL); -- violation
+ROLLBACK;
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES (5);
+INSERT INTO ft8_2pc VALUES (5);
+SAVEPOINT S1;
+INSERT INTO ft7_2pc VALUES (6);
+INSERT INTO ft8_2pc VALUES (6);
+ROLLBACK TO S1;
+COMMIT;
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+RELEASE SAVEPOINT S1;
+
+-- When set to 'disabled', we can commit it
+SET foreign_twophase_commit TO 'disabled';
+BEGIN;
+INSERT INTO ft7_2pc VALUES(8);
+INSERT INTO ft8_2pc VALUES(8);
+COMMIT; -- success
+SELECT * FROM ft7_2pc;
+SELECT * FROM ft8_2pc;
+
+SET foreign_twophase_commit TO 'required';
+
+-- Commit and rollback foreign transactions that are part of
+-- prepare transaction.
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+COMMIT PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+BEGIN;
+INSERT INTO ft7_2pc VALUES(9);
+INSERT INTO ft8_2pc VALUES(9);
+PREPARE TRANSACTION 'gx1';
+ROLLBACK PREPARED 'gx1';
+SELECT * FROM ft8_2pc;
+
+-- No entry remained
+SELECT count(*) FROM pg_foreign_xacts;
diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml
index eab2cc9378..3ea3ce9335 100644
--- a/doc/src/sgml/postgres-fdw.sgml
+++ b/doc/src/sgml/postgres-fdw.sgml
@@ -477,6 +477,43 @@ OPTIONS (ADD password_required 'false');
    </para>
 
   </sect3>
+
+  <sect3>
+   <title>Transaction Management Options</title>
+
+   <para>
+    By default, if the transaction involves with multiple remote server,
+    each transaction on remote server is committed or aborted independently.
+    Some of transactions may fail to commit on remote server while other
+    transactions commit successfully. This may be overridden using
+    following option:
+   </para>
+
+   <variablelist>
+
+    <varlistentry>
+     <term><literal>two_phase_commit</literal></term>
+     <listitem>
+      <para>
+       This option controls whether <filename>postgres_fdw</filename> allows
+       to use two-phase-commit when transaction commits. This option can
+       only be specified for foreign servers, not per-table.
+       The default is <literal>false</literal>.
+      </para>
+
+      <para>
+       If this option is enabled, <filename>postgres_fdw</filename> prepares
+       transaction on remote server and <productname>PostgreSQL</productname>
+       keeps track of the distributed transaction.
+       <xref linkend="guc-max-prepared-foreign-transactions"/> must be set more
+       than 1 on local server and <xref linkend="guc-max-prepared-transactions"/>
+       must set to more than 1 on remote server.
+      </para>
+     </listitem>
+    </varlistentry>
+
+   </variablelist>
+  </sect3>
  </sect2>
 
  <sect2>
@@ -504,6 +541,14 @@ OPTIONS (ADD password_required 'false');
    managed by creating corresponding remote savepoints.
   </para>
 
+  <para>
+   <filename>postgrs_fdw</filename> uses two-phase commit protocol during
+   transaction commits or aborts when the atomic commit of distributed
+   transaction (see <xref linkend="atomic-commit"/>) is required. So the remote
+   server should set <xref linkend="guc-max-prepared-transactions"/> more
+   than one so that it can prepare the remote transaction.
+  </para>
+
   <para>
    The remote transaction uses <literal>SERIALIZABLE</literal>
    isolation level when the local transaction has <literal>SERIALIZABLE</literal>
-- 
2.23.0

From 2dca99e2d5fffb629b8b42ffbd4f31c2f63a43da Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Tue, 19 May 2020 15:16:02 +0900
Subject: [PATCH v22 2/6] Support atomic commit among multiple foreign servers.

---
 src/backend/access/Makefile                   |    2 +-
 src/backend/access/fdwxact/Makefile           |   17 +
 src/backend/access/fdwxact/README             |  110 +
 src/backend/access/fdwxact/fdwxact.c          | 2767 +++++++++++++++++
 src/backend/access/fdwxact/launcher.c         |  560 ++++
 src/backend/access/fdwxact/resolver.c         |  436 +++
 src/backend/access/rmgrdesc/Makefile          |    1 +
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   58 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    6 +-
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   56 +
 src/backend/access/transam/xact.c             |   29 +-
 src/backend/access/transam/xlog.c             |   34 +-
 src/backend/catalog/system_views.sql          |    3 +
 src/backend/commands/copy.c                   |    6 +
 src/backend/commands/foreigncmds.c            |   30 +
 src/backend/executor/execPartition.c          |    8 +
 src/backend/executor/nodeForeignscan.c        |   24 +
 src/backend/executor/nodeModifyTable.c        |    6 +
 src/backend/foreign/foreign.c                 |   55 +
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/pgstat.c               |   18 +
 src/backend/postmaster/postmaster.c           |   15 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   46 +
 src/backend/storage/lmgr/lwlocknames.txt      |    3 +
 src/backend/storage/lmgr/proc.c               |    8 +
 src/backend/tcop/postgres.c                   |   14 +
 src/backend/utils/misc/guc.c                  |   79 +
 src/backend/utils/misc/postgresql.conf.sample |   16 +
 src/backend/utils/probes.d                    |    2 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/fdwxactdesc.c              |    1 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |  162 +
 src/include/access/fdwxact_launcher.h         |   28 +
 src/include/access/fdwxact_resolver.h         |   23 +
 src/include/access/fdwxact_xlog.h             |   54 +
 src/include/access/resolver_internal.h        |   63 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xact.h                     |    7 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   22 +
 src/include/foreign/fdwapi.h                  |   12 +
 src/include/foreign/foreign.h                 |    1 +
 src/include/pgstat.h                          |    6 +
 src/include/storage/proc.h                    |   11 +
 src/include/storage/procarray.h               |    5 +
 src/include/utils/guc_tables.h                |    2 +
 src/test/regress/expected/rules.out           |    7 +
 55 files changed, 4822 insertions(+), 17 deletions(-)
 create mode 100644 src/backend/access/fdwxact/Makefile
 create mode 100644 src/backend/access/fdwxact/README
 create mode 100644 src/backend/access/fdwxact/fdwxact.c
 create mode 100644 src/backend/access/fdwxact/launcher.c
 create mode 100644 src/backend/access/fdwxact/resolver.c
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 120000 src/bin/pg_waldump/fdwxactdesc.c
 create mode 100644 src/include/access/fdwxact.h
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile
index 0880e0a8bb..49480dd039 100644
--- a/src/backend/access/Makefile
+++ b/src/backend/access/Makefile
@@ -9,6 +9,6 @@ top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS	    = brin common gin gist hash heap index nbtree rmgrdesc spgist \
-			  table tablesample transam
+			  table tablesample transam fdwxact
 
 include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/Makefile b/src/backend/access/fdwxact/Makefile
new file mode 100644
index 0000000000..0207a66fb4
--- /dev/null
+++ b/src/backend/access/fdwxact/Makefile
@@ -0,0 +1,17 @@
+#-------------------------------------------------------------------------
+#
+# Makefile--
+#    Makefile for access/fdwxact
+#
+# IDENTIFICATION
+#    src/backend/access/fdwxact/Makefile
+#
+#-------------------------------------------------------------------------
+
+subdir = src/backend/access/fdwxact
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+
+OBJS = fdwxact.o resolver.o launcher.o
+
+include $(top_srcdir)/src/backend/common.mk
diff --git a/src/backend/access/fdwxact/README b/src/backend/access/fdwxact/README
new file mode 100644
index 0000000000..d9f08f4cfa
--- /dev/null
+++ b/src/backend/access/fdwxact/README
@@ -0,0 +1,110 @@
+src/backend/access/fdwxact/README
+
+Atomic Commit for Distributed Transactions
+===========================================
+
+The atomic commit feature enables us to commit and rollback either all of
+foreign servers or nothing. This ensures that the database data is always left
+in a consistent state in term of federated database.
+
+
+Commit Sequence of Global Transactions
+---------------------------------------
+
+We employee two-phase commit protocol to achieve commit among all foreign
+servers atomically. The sequence of distributed transaction commit consists
+of the following four steps:
+
+1. Foreign Server Registration
+During executor node initialization, accessed foreign servers are registered
+to the list FdwXactAtomicCommitParticipants, which is maintained by
+PostgreSQL's the global transaction manager (GTM), as a distributed transaction
+participant. The registered foreign transactions are tracked until the end of
+transaction.
+
+2. Pre-Commit phase (1st phase of two-phase commit)
+We record the corresponding WAL indicating that the foreign server is involved
+with the current transaction before doing PREPARE each foreign transactions.
+Thus in case we loose connectivity to the foreign server or crash ourselves,
+we will remember that we might have prepared transaction on the foreign
+server, and try to resolve it when connectivity is restored or after crash
+recovery.
+
+The two-phase commit is required only if the transaction modified two or more
+servers including the local node. In other case, we can commit them at this
+step by calling CommitForeignTransaction() API and no need further operation.
+
+After that we prepare all foreign transactions by calling
+PrepareForeignTransaction() API. If we failed on any of them we change to
+rollback, therefore at this time some participants might be prepared whereas
+some are not prepared. The former foreign transactions need to be resolved
+using pg_resolve_foreign_xact() manually and the latter ends transaction
+in one-phase by calling RollbackForeignTransaction() API.
+
+3. Commit locally
+Once we've prepared all of them, commit the transaction locally.
+
+4. Post-Commit Phase (2nd phase of two-phase commit)
+The steps so far are done by the backend process committing the transaction but
+this resolution step (commit or rollback) is done by the foreign transaction
+resolver process. The backend process inserts itself to the wait queue, and
+then wake up the resolver process (or request to launch new one if necessary).
+The resolver process enqueue the waiter and fetch the distributed transaction
+information that the backend is waiting for. Once all foreign transaction are
+committed or rollbacked the resolver process wake up the waiter.
+
+
+Foreign Data Wrapper Callbacks for Transaction Management
+-----------------------------------------------------------
+
+The core GTM manages the status of individual foreign transactions and calls
+transaction management callback functions according to its status. Each
+callback functions PrepareForeignTransaction, CommitForeignTransaction and
+RollbackForeignTransaction is responsible for PREPARE, COMMIT or ROLLBACK
+the transaction on the foreign server, respectively.
+FdwXactRslvState->flags could contain FDWXACT_FLAG_ONEPHASE, meaning FDW can
+commit or rollback the foreign transaction in one-phase. On failure during
+processing a foreign transaction, FDW needs to raise an error. However, FDW
+must accept ERRCODE_UNDEFINED_OBJECT error during committing or rolling back a
+foreign transaction, because there is a race condition that the coordinator
+could crash in time between the resolution is completed and writing the WAL
+removing the FdwXact entry.
+
+
+Foreign Transactions Status
+----------------------------
+
+Every foreign transactions will have an FdwXact entry. When preparing a foreign
+transaction a FdwXact entry of which status starts from FDWXACT_STATUS_PREPARING
+are created with WAL logging. The status changes to FDWXACT_STATUS_PREPARED
+after the foreign transaction is prepared and it changes to
+FDWXACT_STATUS_COMMITTING and FDWXACT_STATUS_ABORTING before the foreign
+transaction is committed and aborted by FDW callback functions respectively.
+FdwXact entry is removed once the foreign transaction is resolved with WAL
+logging.
+
+FdwXact entries recovered during the recovery are marked as in-doubt if the
+corresponding local transaction is not prepared transaction. The initial
+status is FDWXACT_STATUS_PREPARED(*1). Because the foreign transaction was
+being processed we cannot know the exact status. So we regard it as PREPARED
+for safety.
+
+The foreign transaction status transition is illustrated by the following graph
+describing the FdwXact->status:
+
+ +----------------------------------------------------+
+ |                     PREPARING                      |----+
+ +----------------------------------------------------+    |
+                          |                                |
+                          v                                |
+ +----------------------------------------------------+    |
+ |                    PREPARED(*1)                    |    | (*2)
+ +----------------------------------------------------+    |
+           |                               |               |
+           v                               v               |
+ +--------------------+          +--------------------+    |
+ |   COMMITTING(*1)   |          |    ABORTING(*1)    |<---+
+ +--------------------+          +--------------------+
+
+(*1) Recovered FdwXact entries starts with PREPARED
+(*2) Paths when an error occurrs during preparing
diff --git a/src/backend/access/fdwxact/fdwxact.c b/src/backend/access/fdwxact/fdwxact.c
new file mode 100644
index 0000000000..86630972c0
--- /dev/null
+++ b/src/backend/access/fdwxact/fdwxact.c
@@ -0,0 +1,2767 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact.c
+ *		PostgreSQL global transaction manager for foreign servers.
+ *
+ * To achieve commit among all foreign servers atomically, we employee
+ * two-phase commit protocol, which is a type of atomic commitment
+ * protocol(ACP). The basic strategy is that we prepare all of the remote
+ * transactions before committing locally and commit them after committing
+ * locally.
+ *
+ * Two-phase commit protocol is used when the transaction modified two or
+ * more servers including the local node.  If two-phase commit protocol
+ * is not required all foreign transactions are committed at pre-commit
+ * phase.
+ *
+ * During executor node initialization, they can register the foreign server
+ * by calling either RegisterFdwXactByRelId() or RegisterFdwXactByServerId()
+ * to participate it to a group for global commit.  The foreign servers are
+ * registered if FDW has both CommitForeignTransaction API and
+ * RollbackForeignTransaction API.  Registered participant servers are
+ * identified by OIDs of foreign server and user.
+ *
+ * During pre-commit of local transaction, we prepare the transaction on
+ * all foreign servers.  And after committing or rolling back locally,
+ * we notify the resolver process and tell it to commit or rollback those
+ * transactions. If we ask to commit, we also tell to notify us when
+ * it's done, so that we can wait interruptibly to finish, and so that
+ * we're not trying to locally do work that might fail after foreign
+ * transaction are committed.
+ *
+ * The best performing way to manage the waiting backends is to have a
+ * queue of waiting backends, so that we can avoid searching the through all
+ * foreign transactions each time we receive a request.  We have one queue
+ * of which elements are ordered by the timestamp when they expect to be
+ * processed.  Before waiting for foreign transactions being resolved the
+ * backend enqueues with the timestamp when they expects to be processed.
+ * On failure, it enqueues again with new timestamp (last timestamp +
+ * foreign_xact_resolution_interval).
+ *
+ * If any network failure, server crash occurs or user stopped waiting
+ * prepared foreign transactions are left in in-doubt state (aka. in-doubt
+ * transaction).  Foreign transactions in in-doubt state are not resolved
+ * automatically so must be processed manually using by
+ * pg_resovle_foreign_xact() function.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed, the corresponding FdwXact
+ * entry is update.  To avoid holding the lock during transaction processing
+ * which may take an unpredicatable time the in-memory data of foreign
+ * transaction follows a locking model based on four linked concepts:
+ *
+ * * All FdwXact fields except for indoubt, inprocessing and status are protected
+ *   by FdwXactLock.  These three fields are protected by its mutex.
+ * * Setting held_by of an FdwXact entry means to own the FdwXact entry, which
+ *   prevent it from updated and removed by concurrent processes.
+ * * The FdwXact whose inprocessing is true is also not processed or removed
+ *   by concurrent processes.
+ * * A process who is going to process foreign transaction needs to hold its
+ *   FdwXact entry in advance.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXact
+ *   with entries marked with fdwxact->inredo and fdwxact->ondisk.  FdwXact file
+ *   data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->fdwxacts.
+ *   We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->fdwxacts entries that
+ *   have fdwxact->inredo set and are behind the redo_horizon.  We save
+ *   them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->fdwxacts.  If
+ *   fdwxact->ondisk is true, the corresponding entry from the disk is
+ *   additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *   fdwxact->inredo entries that have not made it to disk.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *    src/backend/access/fdwxact/fdwxact.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_xlog.h"
+#include "access/resolver_internal.h"
+#include "access/heapam.h"
+#include "access/htup_details.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
+#include "catalog/pg_type.h"
+#include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
+#include "funcapi.h"
+#include "libpq/pqsignal.h"
+#include "miscadmin.h"
+#include "parser/parsetree.h"
+#include "pg_trace.h"
+#include "pgstat.h"
+#include "storage/fd.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/proc.h"
+#include "storage/procarray.h"
+#include "storage/pmsignal.h"
+#include "storage/shmem.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
+#include "utils/memutils.h"
+#include "utils/ps_status.h"
+#include "utils/rel.h"
+#include "utils/snapmgr.h"
+
+/* Foreign twophase commit is enabled and requested by user */
+#define IsForeignTwophaseCommitRequested() \
+	 (foreign_twophase_commit > FOREIGN_TWOPHASE_COMMIT_DISABLED)
+
+/* Check the FdwXactParticipant is capable of two-phase commit  */
+#define ServerSupportTransactionCallack(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->commit_foreign_xact_fn != NULL)
+#define SeverSupportTwophaseCommit(fdw_part) \
+	(((FdwXactParticipant *)(fdw_part))->prepare_foreign_xact_fn != NULL)
+
+/*
+ * Structure to bundle the foreign transaction participant.  This struct
+ * is created at the beginning of execution for each foreign servers and
+ * is used until the end of transaction where we cannot look at syscaches.
+ * Therefore, this is allocated in the TopTransactionContext.
+ */
+typedef struct FdwXactParticipant
+{
+	/*
+	 * Pointer to a FdwXact entry in the global array. NULL if the entry is
+	 * not inserted yet but this is registered as a participant.
+	 */
+	FdwXact		fdwxact;
+
+	/* Foreign server and user mapping info, passed to callback routines */
+	ForeignServer *server;
+	UserMapping *usermapping;
+
+	/* Transaction identifier used for PREPARE */
+	char	   *fdwxact_id;
+
+	/* true if modified the data on the server */
+	bool		modified;
+
+	/* Callbacks for foreign transaction */
+	PrepareForeignTransaction_function prepare_foreign_xact_fn;
+	CommitForeignTransaction_function commit_foreign_xact_fn;
+	RollbackForeignTransaction_function rollback_foreign_xact_fn;
+	GetPrepareId_function get_prepareid_fn;
+} FdwXactParticipant;
+
+/*
+ * List of foreign transactions involved in the transaction.  A member of
+ * participants may not support transaction callbacks: commit, rollback and
+ * prepare.  If a member of participants doesn't support any transaction
+ * callbacks, i.g. ServerSupportTransactionCallack() returns false,
+ * we don't end its transaction.
+ *
+ * FdwXactParticipants_tmp is used to update FdwXactParticipants atomically
+ * when executing COMMIT/ROLLBACK PREPARED command.  In COMMIT PREPARED case,
+ * we don't want to rollback foreign transactions even if an error occurs,
+ * because the local prepared transaction never turn over rollback in that
+ * case.  However, preparing FdwXactParticipants might be lead an error
+ * because of calling palloc() inside.  So we prepare FdwXactParticipants in
+ * two phase.  In the first phase, PrepareFdwXactParticipants(), we collect
+ * all foreign transactions associated with the local prepared transactions
+ * and kept them in FdwXactParticipants_tmp.  Even if an error occurs during
+ * that, we don't rollback them.  In the second phase, SetFdwXactParticipants(),
+ * we replace FdwXactParticipants_tmp with FdwXactParticipants and hold them.
+ *
+ * FdwXactLocalXid is the local transaction id associated with FdwXactParticipants.
+ */
+static List *FdwXactParticipants = NIL;
+static List *FdwXactParticipants_tmp = NIL;
+static TransactionId FdwXactLocalXid = InvalidTransactionId;
+
+/*
+ * True is the current transaction needs to be committed together with
+ * foreign servers.
+ */
+static bool ForeignTwophaseCommitIsRequired = false;
+
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes database oid,
+ * xid, foreign server oid and user oid separated by '_'.
+ *
+ * Since FdwXact stat file is created per foreign transaction in a
+ * distributed transaction and the xid of unresolved distributed
+ * transaction never reused, the name is fairly enough to ensure
+ * uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8 + 1 + 8)
+#define FdwXactFilePath(path, dbid, xid, serverid, userid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X_%08X_%08X", \
+			 dbid, xid, serverid, userid)
+
+/* Guc parameters */
+int			max_prepared_foreign_xacts = 0;
+int			max_foreign_xact_resolvers = 0;
+int			foreign_twophase_commit = FOREIGN_TWOPHASE_COMMIT_DISABLED;
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+static void register_fdwxact(Oid serverid, Oid userid, bool modified);
+static void FdwXactParticipantEndTransaction(FdwXactParticipant *fdw_part, bool onephase,
+											 bool for_commit);
+static bool checkForeignTwophaseCommitRequired(void);
+static FdwXact FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part);
+static FdwXact insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, Oid umid, char *fdwxact_id);
+static void FdwXactPrepareForeignTransactions(bool prepare_all);
+static void FdwXactResolveOneFdwXact(FdwXact fdwxact);
+static void FdwXactComputeRequiredXmin(void);
+static void FdwXactCancelWait(void);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool give_warnings);
+static void FdwXactQueueInsert(PGPROC *waiter);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void ForgetAllFdwXactParticipants(bool mark_indoubt);
+static char *ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							 Oid userid);
+static void RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+							  Oid userid, bool giveWarning);
+static void RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+								Oid userid, void *content, int len);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(Oid dbid, TransactionId local_xid,
+								  Oid serverid, Oid userid,
+								  XLogRecPtr insert_start_lsn,
+								  bool from_disk);
+static TransactionId FdwXactDetermineTransactionFate(TransactionId xid);
+static int	get_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+						bool hold);
+static char *get_fdwxact_identifier(FdwXactParticipant *fdw_part,
+									TransactionId xid);
+static void remove_fdwxact(FdwXact fdwxact);
+static FdwXactParticipant *create_fdwxact_participant(Oid serverid, Oid userid,
+													  FdwRoutine *routine);
+
+#ifdef USE_ASSERT_CHECKING
+static bool FdwXactQueueIsOrderedByTimestamp(void);
+#endif
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, fdwxacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXact)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXact		fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_fdwxacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXact)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, fdwxacts) +
+					  sizeof(FdwXact) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&fdwxacts[cnt].mutex);
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
+
+/*
+ * Remember accessed foreign transaction. Both RegisterFdwXactByRelId and
+ * RegisterFdwXactByServerId are called by executor during initialization.
+ */
+void
+RegisterFdwXactByRelId(Oid relid, bool modified)
+{
+	Relation	rel;
+	Oid			serverid;
+	Oid			userid;
+
+	rel = relation_open(relid, NoLock);
+	serverid = GetForeignServerIdByRelId(relid);
+	userid = rel->rd_rel->relowner ? rel->rd_rel->relowner : GetUserId();
+	relation_close(rel, NoLock);
+
+	register_fdwxact(serverid, userid, modified);
+}
+
+void
+RegisterFdwXactByServerId(Oid serverid, bool modified)
+{
+	register_fdwxact(serverid, GetUserId(), modified);
+}
+
+/*
+ * Register given foreign transaction identified by given arguments as
+ * a participant of the transaction. The foreign transaction identified
+ * by given server id and user id.
+ */
+static void
+register_fdwxact(Oid serverid, Oid userid, bool modified)
+{
+	FdwXactParticipant *fdw_part;
+	MemoryContext old_ctx;
+	FdwRoutine *routine;
+	ListCell   *lc;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (fdw_part->server->serverid == serverid &&
+			fdw_part->usermapping->userid == userid)
+		{
+			/* The foreign server is already registered, return */
+			fdw_part->modified |= modified;
+			return;
+		}
+	}
+
+	routine = GetFdwRoutineByServerId(serverid);
+
+	/*
+	 * Participant's information is also needed at the end of a transaction,
+	 * where system cache are not available. Save it in TopTransactionContext
+	 * so that these can live until the end of transaction.
+	 */
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	fdw_part = create_fdwxact_participant(serverid, userid, routine);
+	fdw_part->modified = modified;
+
+	/* Add to the participants list */
+	FdwXactParticipants = lappend(FdwXactParticipants, fdw_part);
+
+	pfree(routine);
+
+	/* Revert back the context */
+	MemoryContextSwitchTo(old_ctx);
+}
+
+/* Return palloc'd FdwXactParticipant variable */
+static FdwXactParticipant *
+create_fdwxact_participant(Oid serverid, Oid userid, FdwRoutine *routine)
+{
+	FdwXactParticipant *fdw_part;
+	ForeignServer *foreign_server;
+	UserMapping *user_mapping;
+
+	foreign_server = GetForeignServer(serverid);
+	user_mapping = GetUserMapping(userid, serverid);
+
+	fdw_part = (FdwXactParticipant *) palloc(sizeof(FdwXactParticipant));
+
+	fdw_part->fdwxact = NULL;
+	fdw_part->server = foreign_server;
+	fdw_part->usermapping = user_mapping;
+	fdw_part->fdwxact_id = NULL;
+	fdw_part->modified = false;
+	fdw_part->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
+	fdw_part->commit_foreign_xact_fn = routine->CommitForeignTransaction;
+	fdw_part->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdw_part->get_prepareid_fn = routine->GetPrepareId;
+
+	return fdw_part;
+}
+
+/*
+ * Prepare all foreign transactions if foreign twophase commit is required.
+ * When foreign twophase commit is enabled, the behavior depends on the value
+ * of foreign_twophase_commit; when 'required' we strictly require for all
+ * foreign servers' FDW to support two-phase commit protocol and ask them to
+ * prepare foreign transactions, and when 'disabled' we ask all foreign servers
+ * to commit foreign transaction in one-phase. If we failed to commit any of
+ * them we change to aborting.
+ *
+ * Note that non-modified foreign servers always can be committed without
+ * preparation.
+ */
+void
+PreCommit_FdwXact(void)
+{
+	ListCell   *lc;
+	bool		need_twophase_commit;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Set the local transaction id */
+	FdwXactLocalXid = GetTopTransactionId();
+
+	/*
+	 * We don't support foreign twophase commit in single user mode. Commit
+	 * each transaction in one-phase and reset the participant list.
+	 */
+	if (!IsUnderPostmaster)
+	{
+		foreach(lc, FdwXactParticipants)
+		{
+			FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+			/* Commit the foreign transaction in one-phase */
+			if (ServerSupportTransactionCallack(fdw_part))
+				FdwXactParticipantEndTransaction(fdw_part, true, true);
+		}
+
+		list_free(FdwXactParticipants);
+		FdwXactParticipants = NIL;
+		return;
+	}
+
+	/*
+	 * Check if we need to use foreign twophase commit. It's always false if
+	 * foreign twophase commit is disabled.
+	 */
+	need_twophase_commit = checkForeignTwophaseCommitRequired();
+
+	/*
+	 * Prepare foreign transactions on foreign servers that support two-phase
+	 * commit.
+	 */
+	if (need_twophase_commit)
+	{
+		FdwXactPrepareForeignTransactions(false);
+		ForeignTwophaseCommitIsRequired = true;
+	}
+
+	/*
+	 * Commit other foreign transactions and delete the participant entry from
+	 * the list.
+	 */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		/*
+		 * Skip already prepared foreign transactions. Note that we keep those
+		 * FdwXactParticipants until the end of the transaction.
+		 */
+		if (fdw_part->fdwxact)
+			continue;
+
+		/* Delete non-transaction-support participants */
+		if (!ServerSupportTransactionCallack(fdw_part))
+		{
+			FdwXactParticipants = foreach_delete_current(FdwXactParticipants, lc);
+			continue;
+		}
+
+		/* Commit the foreign transaction in one-phase */
+		FdwXactParticipantEndTransaction(fdw_part, true, true);
+
+		/* Transaction successfully committed delete from the participant list */
+		FdwXactParticipants = foreach_delete_current(FdwXactParticipants, lc);
+	}
+}
+
+/*
+ * Return true if the current transaction modifies data on two or more servers
+ * in FdwXactParticipants and local server itself.
+ */
+static bool
+checkForeignTwophaseCommitRequired(void)
+{
+	ListCell   *lc;
+	bool		need_twophase_commit;
+	bool		have_notwophase = false;
+	int			nserverswritten = 0;
+
+	if (!IsForeignTwophaseCommitRequested())
+		return false;
+
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (!fdw_part->modified)
+			continue;
+
+		if (!SeverSupportTwophaseCommit(fdw_part))
+			have_notwophase = true;
+
+		nserverswritten++;
+	}
+
+	/* Did we modify the local non-temporary data? */
+	if ((MyXactFlags & XACT_FLAGS_WROTENONTEMPREL) != 0)
+		nserverswritten++;
+
+	if (nserverswritten <= 1)
+		return false;
+
+	if (foreign_twophase_commit == FOREIGN_TWOPHASE_COMMIT_REQUIRED)
+	{
+		/*
+		 * In 'required' case, we require for all modified server to support
+		 * two-phase commit.
+		 */
+		need_twophase_commit = (nserverswritten >= 2);
+	}
+
+	/*
+	 * If foreign two phase commit is required then all foreign serves must be
+	 * capable of doing two-phase commit
+	 */
+	if (need_twophase_commit)
+	{
+		/* Parameter check */
+		if (max_prepared_foreign_xacts == 0)
+			ereport(ERROR,
+					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					 errmsg("foreign two-phase commit is required but prepared foreign transactions are disabled"),
+					 errhint("Set max_prepared_foreign_transactions to a nonzero value.")));
+
+		if (max_foreign_xact_resolvers == 0)
+			ereport(ERROR,
+					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					 errmsg("foreign two-phase commit is required but prepared foreign transactions are disabled"),
+					 errhint("Set max_foreign_transaction_resolvers to a nonzero value.")));
+
+		if (have_notwophase)
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					 errmsg("cannot process a distributed transaction that has operated on a foreign server that does not support two-phase commit protocol"),
+					 errdetail("foreign_twophase_commit is \'required\' but the transaction has some foreign servers which are not capable of two-phase commit")));
+	}
+
+	return need_twophase_commit;
+}
+
+/*
+ * The routine for committing or rolling back the given transaction participant.
+ */
+static void
+FdwXactParticipantEndTransaction(FdwXactParticipant *fdw_part, bool onephase,
+								 bool for_commit)
+{
+	FdwXactRslvState state;
+
+	Assert(fdw_part->commit_foreign_xact_fn);
+	Assert(fdw_part->rollback_foreign_xact_fn);
+
+	state.xid = FdwXactLocalXid;
+	state.server = fdw_part->server;
+	state.usermapping = fdw_part->usermapping;
+	state.fdwxact_id = onephase ? NULL : fdw_part->fdwxact_id;
+	state.flags = onephase ? FDWXACT_FLAG_ONEPHASE : 0;
+
+	if (for_commit)
+	{
+		fdw_part->commit_foreign_xact_fn(&state);
+		elog(DEBUG1, "successfully committed the foreign transaction for server %u user %u",
+			 fdw_part->usermapping->serverid,
+			 fdw_part->usermapping->userid);
+	}
+	else
+	{
+		fdw_part->rollback_foreign_xact_fn(&state);
+		elog(DEBUG1, "successfully rolled back the foreign transaction for server %u user %u",
+			 fdw_part->usermapping->serverid,
+			 fdw_part->usermapping->userid);
+	}
+}
+
+/*
+ * Insert FdwXact entries and prepare foreign transactions. Before inserting
+ * FdwXact entry we call get_preparedid callback to get a transaction
+ * identifier from FDW.
+ *
+ * We still can change to rollback here on failure. If any error occurs, we
+ * rollback non-prepared foreign transactions.
+ */
+static void
+FdwXactPrepareForeignTransactions(bool prepare_all)
+{
+	ListCell   *lc;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Loop over the foreign connections */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+		FdwXactRslvState state;
+		FdwXact		fdwxact;
+
+		if (!SeverSupportTwophaseCommit(fdw_part))
+			continue;
+
+		if (!prepare_all && !fdw_part->modified)
+			continue;
+
+		/* Get prepared transaction identifier */
+		fdw_part->fdwxact_id = get_fdwxact_identifier(fdw_part, FdwXactLocalXid);
+		Assert(fdw_part->fdwxact_id);
+
+		/*
+		 * Insert the foreign transaction entry with the
+		 * FDWXACT_STATUS_PREPARING status. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 *
+		 * If we prepare the transaction on the foreign server before
+		 * persisting the information to the disk and crash in-between these
+		 * two steps, we will lost the prepared transaction on the foreign
+		 * server and will not be able to resolve it after the crash recovery.
+		 * Hence persist first then prepare.
+		 */
+		fdwxact = FdwXactInsertFdwXactEntry(FdwXactLocalXid, fdw_part);
+
+		/*
+		 * Prepare the foreign transaction.
+		 *
+		 * Between FdwXactInsertFdwXactEntry call till this backend hears
+		 * acknowledge from foreign server, the backend may abort the local
+		 * transaction (say, because of a signal).
+		 */
+		state.xid = FdwXactLocalXid;
+		state.server = fdw_part->server;
+		state.usermapping = fdw_part->usermapping;
+		state.fdwxact_id = pstrdup(fdw_part->fdwxact_id);
+		fdw_part->prepare_foreign_xact_fn(&state);
+
+		/* succeeded, update status */
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwxact->status = FDWXACT_STATUS_PREPARED;
+		SpinLockRelease(&fdwxact->mutex);
+	}
+}
+
+/*
+ * This function is used to create new foreign transaction entry before an FDW
+ * prepares and commit/rollback. The function adds the entry to WAL and it will
+ * be persisted to the disk under pg_fdwxact directory when checkpoint.
+ */
+static FdwXact
+FdwXactInsertFdwXactEntry(TransactionId xid, FdwXactParticipant *fdw_part)
+{
+	FdwXact		fdwxact;
+	FdwXactOnDiskData *fdwxact_file_data;
+	MemoryContext old_context;
+	int			data_len;
+
+	old_context = MemoryContextSwitchTo(TopTransactionContext);
+
+	/*
+	 * Enter the foreign transaction in the shared memory structure.
+	 */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdw_part->server->serverid,
+							 fdw_part->usermapping->userid,
+							 fdw_part->usermapping->umid, fdw_part->fdwxact_id);
+	fdwxact->status = FDWXACT_STATUS_PREPARING;
+	fdwxact->held_by = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	fdw_part->fdwxact = fdwxact;
+	MemoryContextSwitchTo(old_context);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactOnDiskData, fdwxact_id);
+	data_len = data_len + strlen(fdw_part->fdwxact_id) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactOnDiskData *) palloc0(data_len);
+	fdwxact_file_data->dbid = MyDatabaseId;
+	fdwxact_file_data->local_xid = xid;
+	fdwxact_file_data->serverid = fdw_part->server->serverid;
+	fdwxact_file_data->userid = fdw_part->usermapping->userid;
+	fdwxact_file_data->umid = fdw_part->usermapping->umid;
+	memcpy(fdwxact_file_data->fdwxact_id, fdw_part->fdwxact_id,
+		   strlen(fdw_part->fdwxact_id) + 1);
+
+	/* See note in RecordTransactionCommit */
+	MyProc->delayChkpt = true;
+
+	START_CRIT_SECTION();
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+	XLogFlush(fdwxact->insert_end_lsn);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyProc->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXact
+insert_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+			   Oid umid, char *fdwxact_id)
+{
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+		if (fdwxact->valid &&
+			fdwxact->dbid == dbid &&
+			fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			ereport(ERROR, (errmsg("could not insert a foreign transaction entry"),
+							errdetail("Duplicate entry with transaction id %u, serverid %u, userid %u exists.",
+									  xid, serverid, userid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_fdwxacts < max_prepared_foreign_xacts);
+	FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts++] = fdwxact;
+
+	fdwxact->local_xid = xid;
+	fdwxact->dbid = dbid;
+	fdwxact->serverid = serverid;
+	fdwxact->userid = userid;
+	fdwxact->umid = umid;
+	fdwxact->inprocessing = false;
+	fdwxact->indoubt = false;
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+	memcpy(fdwxact->fdwxact_id, fdwxact_id, strlen(fdwxact_id) + 1);
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXact fdwxact)
+{
+	int			i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		if (FdwXactCtl->fdwxacts[i] == fdwxact)
+			break;
+	}
+
+	/* We did not find the given entry in the array */
+	if (i >= FdwXactCtl->num_fdwxacts)
+		ereport(ERROR,
+				(errmsg("could not remove a foreign transaction entry"),
+				 errdetail("Failed to find entry for xid %u, foreign server %u, and user %u.",
+						   fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+	elog(DEBUG2, "remove fdwxact entry id %s, xid %u db %d user %d",
+		 fdwxact->fdwxact_id, fdwxact->local_xid, fdwxact->dbid,
+		 fdwxact->userid);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_fdwxacts--;
+	FdwXactCtl->fdwxacts[i] = FdwXactCtl->fdwxacts[FdwXactCtl->num_fdwxacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->held_by = InvalidBackendId;
+	fdwxact->indoubt = false;
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.serverid = fdwxact->serverid;
+		record.dbid = fdwxact->dbid;
+		record.xid = fdwxact->local_xid;
+		record.userid = fdwxact->userid;
+
+		/*
+		 * Now writing FdwXact state data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the WAL
+		 * record is inserted could complete without fsync'ing our state file.
+		 * (This is essentially the same kind of race condition as the
+		 * COMMIT-to-clog-write case that RecordTransactionCommit uses
+		 * delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyProc->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and remove
+		 * the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+		XLogFlush(recptr);
+
+		/* Now we can mark ourselves as out of the commit critical section */
+		MyProc->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier. If the given
+ * foreign server's FDW provides getPrepareId callback we return the identifier
+ * returned from it. Otherwise we generate an unique identifier with in the
+ * form of "fx_<random number>_<xid>_<serverid>_<userid> whose length is
+ * less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+get_fdwxact_identifier(FdwXactParticipant *fdw_part, TransactionId xid)
+{
+	char	   *id;
+	int			id_len = 0;
+
+	/*
+	 * If FDW doesn't provide the callback function, generate an unique
+	 * identifier.
+	 */
+	if (!fdw_part->get_prepareid_fn)
+	{
+		char		buf[FDWXACT_ID_MAX_LEN] = {0};
+
+		snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%d_%d",
+				 Abs(random()), xid, fdw_part->server->serverid,
+				 fdw_part->usermapping->userid);
+
+		return pstrdup(buf);
+	}
+
+	/* Get an unique identifier from callback function */
+	id = fdw_part->get_prepareid_fn(xid, fdw_part->server->serverid,
+									fdw_part->usermapping->userid,
+									&id_len);
+
+	if (id == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 (errmsg("foreign transaction identifier is not provided"))));
+
+	/* Check length of foreign transaction identifier */
+	if (id_len > FDWXACT_ID_MAX_LEN)
+	{
+		id[FDWXACT_ID_MAX_LEN] = '\0';
+		ereport(ERROR,
+				(errcode(ERRCODE_NAME_TOO_LONG),
+				 errmsg("foreign transaction identifier \"%s\" is too long",
+						id),
+				 errdetail("Foreign transaction identifier must be less than %d characters.",
+						   FDWXACT_ID_MAX_LEN)));
+	}
+
+	id[id_len] = '\0';
+	return pstrdup(id);
+}
+
+/*
+ * Prepare foreign transactions by PREPARE TRANSACTION command.
+ *
+ * Note that it's possible that the transaction aborts after we prepared some
+ * of participants. In this case we change to rollback and rollback all foreign
+ * transactions.
+ */
+void
+AtPrepare_FdwXact(void)
+{
+	ListCell   *lc;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	/* Set the local transaction id */
+	FdwXactLocalXid = GetTopTransactionId();
+
+	/* Check for an invalid condition */
+	if (!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot PREPARE a distributed transaction when foreign_twophase_commit is \'disabled\'")));
+
+	/*
+	 * Check if there is a server that doesn't support two-phase commit. All involved
+	 * servers need to support two-phase commit as we prepare on them regardless of
+	 * modified or not.
+	 */
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		if (!SeverSupportTwophaseCommit(fdw_part))
+			ereport(ERROR,
+					(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+					 errmsg("cannot PREPARE a distributed transaction which has operated on a foreign server not supporting two-phase commit protocol")));
+	}
+
+	/* Prepare transactions on participating foreign servers. */
+	FdwXactPrepareForeignTransactions(true);
+
+	/*
+	 * We keep prepared foreign transaction participants to rollback them in case
+	 * of failure.
+	 */
+}
+
+void
+PostPrepare_FdwXact(void)
+{
+	/* After preparing the local transaction, we can forget all participants */
+	ForgetAllFdwXactParticipants(false);
+}
+
+/*
+ * Collect all foreign transactions associated with the given xid if it's a prepared
+ * transaction.  Return true if COMMIT PREPARED or ROLLBACK PREPARED needs to wait for
+ * all foreign transactions to be resolved.  The collected foreign transactions are kept
+ * in FdwXactParticipants_tmp. The caller must call SetFdwXactParticipants() later
+ * if this function returns true.
+ */
+bool
+PrepareFdwXactParticipants(TransactionId xid)
+{
+	MemoryContext old_ctx;
+
+	Assert(FdwXactParticipants_tmp == NIL);
+
+	if (!TwoPhaseExists(xid))
+		return false;
+
+	old_ctx = MemoryContextSwitchTo(TopTransactionContext);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXactParticipant *fdw_part;
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		FdwRoutine *routine;
+
+		if (!fdwxact->valid || fdwxact->local_xid != xid)
+			continue;
+
+		routine = GetFdwRoutineByServerId(fdwxact->serverid);
+		fdw_part = create_fdwxact_participant(fdwxact->serverid, fdwxact->userid,
+											  routine);
+		fdw_part->modified = true;
+		fdw_part->fdwxact = fdwxact;
+
+		/* Add to the participants list */
+		FdwXactParticipants_tmp = lappend(FdwXactParticipants_tmp, fdw_part);
+	}
+	LWLockRelease(FdwXactLock);
+
+	MemoryContextSwitchTo(old_ctx);
+
+	/*
+	 * We cannot proceed to commit this prepared transaction when
+	 * foreign_twophase_commit is disabled.
+	 */
+	if (FdwXactParticipants_tmp != NIL &&
+		!IsForeignTwophaseCommitRequested())
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot process a prepared foreign transaction commit when foreign_twophase_commit is \'disabled\'")));
+
+	return (FdwXactParticipants_tmp != NIL);
+}
+
+/*
+ * Set the collected foreign transactions to the participants of this transaction,
+ * and hold them.  This function must be called after CollectFdwXactParticipants().
+ */
+void
+SetFdwXactParticipants(TransactionId xid, bool commit)
+{
+	ListCell   *lc;
+
+	Assert(FdwXactParticipants_tmp != NIL);
+	Assert(FdwXactParticipants == NIL);
+
+	FdwXactLocalXid = xid;
+	FdwXactParticipants = FdwXactParticipants_tmp;
+	FdwXactParticipants_tmp = NIL;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	foreach(lc, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(lc);
+
+		Assert(SeverSupportTwophaseCommit(fdw_part));
+
+		/* Hold the fdwxact entry and set the status */
+		SpinLockAcquire(&fdw_part->fdwxact->mutex);
+		Assert(fdw_part->fdwxact->status == FDWXACT_STATUS_PREPARED);
+		fdw_part->fdwxact->held_by = MyBackendId;
+		fdw_part->fdwxact->status = commit
+			? FDWXACT_STATUS_COMMITTING
+			: FDWXACT_STATUS_ABORTING;
+		SpinLockRelease(&fdw_part->fdwxact->mutex);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+bool
+FdwXactIsForeignTwophaseCommitRequired(void)
+{
+	return ForeignTwophaseCommitIsRequired;
+}
+
+void
+FdwXactCleanupAtProcExit(void)
+{
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+	{
+		LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+		LWLockRelease(FdwXactResolutionLock);
+	}
+}
+
+/*
+ * When the process exits, forget all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllFdwXactParticipants(true);
+}
+
+/*
+ * Wait for its all foreign transactions to be resolved.
+ *
+ * Initially backends start in state FDWXACT_NOT_WAITING and then change
+ * that state to FDWXACT_WAITING before adding ourselves to the wait queue.
+ * During FdwXactResolveForeignTransaction a fdwxact resolver changes the
+ * state to FDWXACT_WAIT_COMPLETE once all foreign transactions are resolved.
+ * This backend then resets its state to FDWXACT_NOT_WAITING.
+ * If a resolver fails to resolve the waiting transaction it moves us to
+ * the retry queue.
+ *
+ * This function is inspired by SyncRepWaitForLSN.
+ */
+void
+FdwXactWaitForResolution(TransactionId wait_xid)
+{
+	char	   *new_status = NULL;
+	const char *old_status;
+
+	Assert(FdwXactCtl != NULL);
+	Assert(TransactionIdIsValid(wait_xid));
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	Assert(MyProc->fdwXactState == FDWXACT_NOT_WAITING);
+
+	/*
+	 * Quick exit if either atomic commit is not requested or we don't have
+	 * any participants.
+	 */
+	if (!IsForeignTwophaseCommitRequested() || FdwXactParticipants == NIL)
+		return;
+
+	/* Set backend status and enqueue itself to the active queue */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	MyProc->fdwXactState = FDWXACT_WAITING;
+	MyProc->fdwXactWaitXid = wait_xid;
+	MyProc->fdwXactNextResolutionTs = GetCurrentTransactionStopTimestamp();
+	FdwXactQueueInsert(MyProc);
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+	LWLockRelease(FdwXactResolutionLock);
+
+	/* Launch a resolver process if not yet, or wake up */
+	FdwXactLaunchOrWakeupResolver();
+
+	/*
+	 * Alter ps display to show waiting for foreign transaction resolution.
+	 */
+	if (update_process_title)
+	{
+		int			len;
+
+		old_status = get_ps_display(&len);
+		new_status = (char *) palloc(len + 31 + 1);
+		memcpy(new_status, old_status, len);
+		sprintf(new_status + len, " waiting for resolution %d", wait_xid);
+		set_ps_display(new_status);
+		new_status[len] = '\0'; /* truncate off "waiting ..." */
+	}
+
+	/* Wait for all foreign transactions to be resolved */
+	for (;;)
+	{
+		/* Must reset the latch before testing state */
+		ResetLatch(MyLatch);
+
+		/*
+		 * Acquiring the lock is not needed, the latch ensures proper
+		 * barriers. If it looks like we're done, we must really be done,
+		 * because once walsender changes the state to FDWXACT_WAIT_COMPLETE,
+		 * it will never update it again, so we can't be seeing a stale value
+		 * in that case.
+		 */
+		if (MyProc->fdwXactState == FDWXACT_WAIT_COMPLETE)
+			break;
+
+		/*
+		 * If a wait for foreign transaction resolution is pending, we can
+		 * neither acknowledge the commit nor raise ERROR or FATAL.  The
+		 * latter would lead the client to believe that the distributed
+		 * transaction aborted, which is not true: it's already committed
+		 * locally. The former is no good either: the client has requested
+		 * committing a distributed transaction, and is entitled to assume
+		 * that a acknowledged commit is also commit on all foreign servers,
+		 * which might not be true. So in this case we issue a WARNING (which
+		 * some clients may be able to interpret) and shut off further output.
+		 * We do NOT reset PorcDiePending, so that the process will die after
+		 * the commit is cleaned up.
+		 */
+		if (ProcDiePending)
+		{
+			ereport(WARNING,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("canceling the wait for resolving foreign transaction and terminating connection due to administrator command"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If a query cancel interrupt arrives we just terminate the wait with
+		 * a suitable warning. The foreign transactions can be orphaned but
+		 * the foreign xact resolver can pick up them and tries to resolve
+		 * them later.
+		 */
+		if (QueryCancelPending)
+		{
+			QueryCancelPending = false;
+			ereport(WARNING,
+					(errmsg("canceling wait for resolving foreign transaction due to user request"),
+					 errdetail("The transaction has already committed locally, but might not have been committed on the foreign server.")));
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * If the postmaster dies, we'll probably never get an
+		 * acknowledgement, because all the wal sender processes will exit. So
+		 * just bail out.
+		 */
+		if (!PostmasterIsAlive())
+		{
+			ProcDiePending = true;
+			whereToSendOutput = DestNone;
+			FdwXactCancelWait();
+			break;
+		}
+
+		/*
+		 * Wait on latch.  Any condition that should wake us up will set the
+		 * latch, so no need for timeout.
+		 */
+		WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH, -1,
+				  WAIT_EVENT_FDWXACT_RESOLUTION);
+	}
+
+	pg_read_barrier();
+
+	Assert(SHMQueueIsDetached(&(MyProc->fdwXactLinks)));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+
+	if (new_status)
+	{
+		set_ps_display(new_status);
+		pfree(new_status);
+	}
+
+	list_free(FdwXactParticipants);
+	FdwXactParticipants = NIL;
+}
+
+/*
+ * Return one backend that connects to my database and is waiting for
+ * resolution.
+ */
+PGPROC *
+FdwXactGetWaiter(TimestampTz now, TimestampTz *nextResolutionTs_p,
+				 TransactionId *waitXid_p)
+{
+	PGPROC	   *proc;
+	bool		found = false;
+
+	Assert(LWLockHeldByMe(FdwXactResolutionLock));
+	Assert(FdwXactQueueIsOrderedByTimestamp());
+
+	/* Initialize variables */
+	*nextResolutionTs_p = -1;
+	*waitXid_p = InvalidTransactionId;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == MyDatabaseId)
+		{
+			if (proc->fdwXactNextResolutionTs <= now)
+			{
+				/* Found a waiting process */
+				found = true;
+				*waitXid_p = proc->fdwXactWaitXid;
+			}
+			else
+				/* Found a waiting process supposed to be processed later */
+				*nextResolutionTs_p = proc->fdwXactNextResolutionTs;
+
+			break;
+		}
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return found ? proc : NULL;
+}
+
+/*
+ * Return true if there are at least one backend in the wait queue. The caller
+ * must hold FdwXactResolutionLock.
+ */
+bool
+FdwXactWaiterExists(Oid dbid)
+{
+	PGPROC	   *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_SHARED));
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->databaseId == dbid)
+			return true;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return false;
+}
+
+/*
+ * Insert the waiter to the wait queue in fdwXactNextResolutoinTs order.
+ */
+static void
+FdwXactQueueInsert(PGPROC *waiter)
+{
+	PGPROC	   *proc;
+
+	Assert(LWLockHeldByMeInMode(FdwXactResolutionLock, LW_EXCLUSIVE));
+
+	proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+
+	while (proc)
+	{
+		if (proc->fdwXactNextResolutionTs < waiter->fdwXactNextResolutionTs)
+			break;
+
+		proc = (PGPROC *) SHMQueuePrev(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	if (proc)
+		SHMQueueInsertAfter(&(proc->fdwXactLinks), &(waiter->fdwXactLinks));
+	else
+		SHMQueueInsertAfter(&(FdwXactRslvCtl->fdwxact_queue), &(waiter->fdwXactLinks));
+}
+
+#ifdef USE_ASSERT_CHECKING
+static bool
+FdwXactQueueIsOrderedByTimestamp(void)
+{
+	PGPROC	   *proc;
+	TimestampTz lastTs;
+
+	proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+								   &(FdwXactRslvCtl->fdwxact_queue),
+								   offsetof(PGPROC, fdwXactLinks));
+	lastTs = 0;
+
+	while (proc)
+	{
+
+		if (proc->fdwXactNextResolutionTs < lastTs)
+			return false;
+
+		lastTs = proc->fdwXactNextResolutionTs;
+
+		proc = (PGPROC *) SHMQueueNext(&(FdwXactRslvCtl->fdwxact_queue),
+									   &(proc->fdwXactLinks),
+									   offsetof(PGPROC, fdwXactLinks));
+	}
+
+	return true;
+}
+#endif
+
+/*
+ * Acquire FdwXactResolutionLock and cancel any wait currently in progress.
+ */
+static void
+FdwXactCancelWait(void)
+{
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(MyProc->fdwXactLinks)))
+		SHMQueueDelete(&(MyProc->fdwXactLinks));
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * In abort case, this function ends foreign transaction participants and possibly
+ * rollback their prepared foreign trasnactions.
+ */
+extern void
+AtEOXact_FdwXact(bool is_commit)
+{
+	ListCell   *lc;
+
+	if (!is_commit)
+	{
+		foreach(lc, FdwXactParticipants)
+		{
+			FdwXactParticipant *fdw_part = lfirst(lc);
+			FdwXact		fdwxact = fdw_part->fdwxact;
+			int			status;
+
+			if (!fdwxact)
+			{
+				/* Rollback foreign transaction in one-phase if supported */
+				if (ServerSupportTransactionCallack(fdw_part))
+					FdwXactParticipantEndTransaction(fdw_part, true, false);
+				continue;
+			}
+
+			/*
+			 * Abort the foreign transaction.  For participants whose status
+			 * is FDWXACT_STATUS_PREPARING, we close the transaction in
+			 * one-phase. In addition, since we are not sure that the
+			 * preparation has been completed on the foreign server, we also
+			 * attempts to rollback the prepared foreign transaction.  Note
+			 * that it's FDWs responsibility that they tolerate
+			 * OBJECT_NOT_FOUND error in abort case.
+			 */
+			SpinLockAcquire(&fdwxact->mutex);
+			status = fdwxact->status;
+			fdwxact->status = FDWXACT_STATUS_ABORTING;
+			SpinLockRelease(&fdwxact->mutex);
+
+			switch (status)
+			{
+				case FDWXACT_STATUS_PREPARING:
+					/* One-phase rollback foreign transaction */
+					FdwXactParticipantEndTransaction(fdw_part, true, false);
+					/* FALLTHROUGH */
+				case FDWXACT_STATUS_PREPARED:
+				case FDWXACT_STATUS_ABORTING:
+					/* One-phase rollback foreign transaction */
+					FdwXactParticipantEndTransaction(fdw_part, false, false);
+					break;
+				case FDWXACT_STATUS_COMMITTING:
+					Assert(false);
+					break;
+			}
+
+			/* Resolution was a success, remove the entry */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			if (fdwxact->ondisk)
+				RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								  fdwxact->serverid, fdwxact->userid,
+								  true);
+			remove_fdwxact(fdwxact);
+			LWLockRelease(FdwXactLock);
+		}
+
+		/* All foreign transaction should be aborted */
+		list_free(FdwXactParticipants);
+		FdwXactParticipants = NIL;
+	}
+
+	ForgetAllFdwXactParticipants(true);
+	ForeignTwophaseCommitIsRequired = false;
+}
+
+/*
+ * Mark my foreign transaction participants as in-doubt and clear
+ * the FdwXactParticipants list.
+ *
+ * If we leave any foreign transaction, update the oldest xmin of unresolved
+ * transaction so that local transaction id of in-doubt transaction is not
+ * truncated.
+ */
+static void
+ForgetAllFdwXactParticipants(bool mark_indoubt)
+{
+	ListCell   *cell;
+	int			nlefts = 0;
+
+	if (FdwXactParticipants == NIL)
+		return;
+
+	foreach(cell, FdwXactParticipants)
+	{
+		FdwXactParticipant *fdw_part = (FdwXactParticipant *) lfirst(cell);
+		FdwXact		fdwxact = fdw_part->fdwxact;
+
+		/* Nothing to do if didn't register FdwXact entry yet */
+		Assert(fdwxact);
+
+		/*
+		 * Unlock and mark a foreign transaction as in-doubt.  Note that there
+		 * is a race condition; the FdwXact entries in FdwXactParticipants
+		 * could be used by other backend before we forget in case where the
+		 * resolver process removes the FdwXact entry and other backend reuses
+		 * it before we forget. So we need to check if the entries are still
+		 * associated with the transaction.  Also we do these check by
+		 * transaction id because these foreign transaction may already be
+		 * held by the resolver.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		if (fdwxact->valid && fdwxact->held_by == MyBackendId)
+		{
+			fdwxact->held_by = InvalidBackendId;
+
+			if (mark_indoubt)
+			{
+				fdwxact->indoubt = true;	/* let resolver to process */
+				nlefts++;
+			}
+		}
+		LWLockRelease(FdwXactLock);
+	}
+
+	/*
+	 * If we left any FdwXact entries, update the oldest local transaction of
+	 * unresolved distributed transaction and take over them to the foreign
+	 * transaction resolver.
+	 */
+	if (nlefts > 0)
+	{
+		Assert(mark_indoubt);
+		elog(DEBUG1, "left %u foreign transactions in in-doubt status", nlefts);
+		FdwXactComputeRequiredXmin();
+		FdwXactLaunchOrWakeupResolver();
+	}
+
+	list_free(FdwXactParticipants);
+	FdwXactParticipants = NIL;
+	FdwXactParticipants_tmp = NIL;
+	FdwXactLocalXid = InvalidTransactionId;
+}
+
+/*
+ * Resolve foreign transactions at the give indexes. If 'waiter' is not NULL,
+ * we release the waiter after we resolved all of the given foreign transactions
+ * On failure we re-enqueue the waiting backend after incremented the next
+ * resolution time.
+ *
+ * The caller must hold the given foreign transactions in advance to prevent
+ * concurrent update.
+ */
+void
+FdwXactResolveFdwXacts(int *fdwxact_idxs, int nfdwxacts, PGPROC *waiter)
+{
+	for (int i = 0; i < nfdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[fdwxact_idxs[i]];
+
+		CHECK_FOR_INTERRUPTS();
+
+		PG_TRY();
+		{
+			FdwXactResolveOneFdwXact(fdwxact);
+		}
+		PG_CATCH();
+		{
+			/*
+			 * Failed to resolve. Re-insert the waiter to the tail of retry
+			 * queue if the waiter is still waiting.
+			 */
+			if (waiter)
+			{
+				LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+				if (waiter->fdwXactState == FDWXACT_WAITING)
+				{
+					SHMQueueDelete(&(waiter->fdwXactLinks));
+					pg_write_barrier();
+					waiter->fdwXactNextResolutionTs =
+						TimestampTzPlusMilliseconds(waiter->fdwXactNextResolutionTs,
+													foreign_xact_resolution_retry_interval);
+					FdwXactQueueInsert(waiter);
+				}
+				LWLockRelease(FdwXactResolutionLock);
+			}
+
+			PG_RE_THROW();
+		}
+		PG_END_TRY();
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		if (fdwxact->ondisk)
+			RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+							  fdwxact->userid, true);
+		remove_fdwxact(fdwxact);
+		LWLockRelease(FdwXactLock);
+	}
+
+	if (!waiter)
+		return;
+
+	/*
+	 * Remove waiter from shmem queue, if not detached yet. The waiter could
+	 * already be detached if user cancelled to wait before resolution.
+	 */
+	LWLockAcquire(FdwXactResolutionLock, LW_EXCLUSIVE);
+	if (!SHMQueueIsDetached(&(waiter->fdwXactLinks)))
+	{
+		TransactionId wait_xid = waiter->fdwXactWaitXid;
+
+		SHMQueueDelete(&(waiter->fdwXactLinks));
+		pg_write_barrier();
+
+		/* Set state to complete */
+		waiter->fdwXactState = FDWXACT_WAIT_COMPLETE;
+
+		/*
+		 * Wake up the waiter only when we have set state and removed from
+		 * queue
+		 */
+		SetLatch(&(waiter->procLatch));
+
+		elog(DEBUG2, "released the proc with xid %u", wait_xid);
+	}
+	else
+		elog(DEBUG2, "the waiter backend had been already detached");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Return true if there is at least one prepared foreign transaction
+ * which matches given arguments.
+ */
+bool
+FdwXactExists(Oid dbid, Oid serverid, Oid userid)
+{
+	int			idx;
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	idx = get_fdwxact(dbid, InvalidTransactionId, serverid, userid,
+					  false);
+	LWLockRelease(FdwXactLock);
+
+	return (idx != -1);
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ *
+ * XXX: we can exclude FdwXact entries whose status is already committing
+ * or aborting.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->local_xid));
+
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->local_xid, agg_xmin))
+			agg_xmin = fdwxact->local_xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+
+/*
+ * Determine whether the foreign transaction associated with the given transaction
+ * id should be committed or rolled back according to the result of the local
+ * transaction.
+ */
+static FdwXactStatus
+FdwXactDetermineTransactionFate(TransactionId xid)
+{
+	/*
+	 * If the local transaction is already committed, commit prepared foreign
+	 * transaction.
+	 */
+	if (TransactionIdDidCommit(xid))
+		return FDWXACT_STATUS_COMMITTING;
+
+	/*
+	 * If the local transaction is already aborted, abort prepared foreign
+	 * transactions.
+	 */
+	else if (TransactionIdDidAbort(xid))
+		return FDWXACT_STATUS_ABORTING;
+
+
+	/*
+	 * The local transaction is not in progress but the foreign transaction is
+	 * not prepared on the foreign server. This can happen when transaction
+	 * failed after registered this entry but before actual preparing on the
+	 * foreign server. So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(xid))
+		return FDWXACT_STATUS_ABORTING;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is about
+	 * to be committed or aborted. This should not happen except for one case
+	 * where the local transaction is prepared and this foreign transaction is
+	 * being resolved manually using by pg_resolve_foreign_xact(). Raise an
+	 * error anyway since we cannot determine the fate of this foreign
+	 * transaction according to the local transaction whose fate is also not
+	 * determined.
+	 */
+	else
+		elog(ERROR,
+			 "cannot resolve the foreign transaction associated with in-process transaction");
+
+	pg_unreachable();
+}
+
+/*
+ * Commit or rollback one prepared foreign transaction.  After resolved
+ * successfully, the FdwXact entry is removed from the shared memory and also
+ * remove the corresponding on-disk file.
+ */
+static void
+FdwXactResolveOneFdwXact(FdwXact fdwxact)
+{
+	FdwXactRslvState state;
+	ForeignServer *server;
+	ForeignDataWrapper *fdw;
+	FdwRoutine *routine;
+
+	Assert(fdwxact != NULL);
+	Assert(fdwxact->held_by != InvalidBackendId || fdwxact->inprocessing);
+
+	if (fdwxact->status != FDWXACT_STATUS_COMMITTING &&
+		fdwxact->status != FDWXACT_STATUS_ABORTING)
+	{
+		FdwXactStatus new_status;
+
+		new_status = FdwXactDetermineTransactionFate(fdwxact->local_xid);
+		Assert(new_status == FDWXACT_STATUS_COMMITTING ||
+			   new_status == FDWXACT_STATUS_ABORTING);
+
+		/* Update the status */
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwxact->status = new_status;
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	server = GetForeignServer(fdwxact->serverid);
+	fdw = GetForeignDataWrapper(server->fdwid);
+	routine = GetFdwRoutine(fdw->fdwhandler);
+
+	/* Prepare resolution state to pass to API */
+	state.xid = fdwxact->local_xid;
+	state.server = server;
+	state.usermapping = GetUserMapping(fdwxact->userid, fdwxact->serverid);
+	state.fdwxact_id = fdwxact->fdwxact_id;
+	state.flags = 0;
+
+	if (fdwxact->status == FDWXACT_STATUS_COMMITTING)
+	{
+		routine->CommitForeignTransaction(&state);
+		elog(DEBUG1, "successfully committed the prepared foreign transaction for server %u user %u",
+			 fdwxact->serverid, fdwxact->userid);
+	}
+	else
+	{
+		routine->RollbackForeignTransaction(&state);
+		elog(DEBUG1, "successfully rolled back the prepared foreign transaction for server %u user %u",
+			 fdwxact->serverid, fdwxact->userid);
+	}
+}
+
+/*
+ * Return a list of FdwXact matched to given arguments. Otherwise return NIL.
+ * The search condition is defined by arguments with valid values for
+ * respective datatypes. 'include_indoubt' and 'include_in_progress' are the
+ * option for that the result includes in-doubt transactions and in-progress
+ * transactions respectively.
+ */
+static int
+get_fdwxact(Oid dbid, TransactionId xid, Oid serverid, Oid userid, bool hold)
+{
+	bool		found = false;
+	int			i;
+
+	Assert(LWLockHeldByMe(FdwXactLock));
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		bool		inprocessing;
+
+		if (!fdwxact->valid)
+			continue;
+
+		SpinLockAcquire(&fdwxact->mutex);
+		inprocessing = fdwxact->inprocessing;
+		SpinLockRelease(&fdwxact->mutex);
+
+		/*
+		 * If we're attempting to hold this entry, skip if it is already held
+		 * or being processed.
+		 */
+		if (hold &&
+			(inprocessing || fdwxact->held_by != InvalidBackendId))
+			continue;
+
+		/* dbid */
+		if (OidIsValid(dbid) && fdwxact->dbid != dbid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid != fdwxact->local_xid)
+			continue;
+
+		/* serverid */
+		if (OidIsValid(serverid) && serverid != fdwxact->serverid)
+			continue;
+
+		/* userid */
+		if (OidIsValid(userid) && fdwxact->userid != userid)
+			continue;
+
+		/* This entry matches the condition */
+
+		if (hold)
+			fdwxact->held_by = MyBackendId;
+
+		found = true;
+		break;
+	}
+
+	return found ? i : -1;
+}
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record in
+		 * FdwXact entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record),
+					   record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXact entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->dbid, record->xid, record->serverid,
+						  record->userid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transaction log record", info);
+
+	return;
+}
+
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactOnDiskData *fdwxact_data = (FdwXactOnDiskData *) buf;
+	FdwXact		fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The status of
+	 * the transaction is set as preparing, since we do not know the exact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction which prepared this foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->local_xid,
+							 fdwxact_data->serverid, fdwxact_data->userid,
+							 fdwxact_data->umid, fdwxact_data->fdwxact_id);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->local_xid,
+		 fdwxact_data->serverid, fdwxact_data->userid,
+		 fdwxact_data->fdwxact_id);
+
+	/*
+	 * Set status as PREPARED and as in-doubt, since we do not know the xact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction that prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;		/* added in redo */
+	fdwxact->indoubt = true;
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXact file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXact entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+void
+FdwXactRedoRemove(Oid dbid, TransactionId xid, Oid serverid,
+				  Oid userid, bool givewarning)
+{
+	FdwXact		fdwxact;
+	int			i;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->dbid == dbid && fdwxact->local_xid == xid &&
+			fdwxact->serverid == serverid && fdwxact->userid == userid)
+			break;
+	}
+
+	if (i >= FdwXactCtl->num_fdwxacts)
+		return;
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  givewarning);
+	remove_fdwxact(fdwxact);
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction, db %u xid %u server %u user %u id %s",
+		 fdwxact->dbid, fdwxact->local_xid, fdwxact->serverid,
+		 fdwxact->userid, fdwxact->fdwxact_id);
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint's redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXacts that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXacts that need to be copied to disk.
+ *
+ * If a FdwXact remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts <= 0)
+		return;					/* nothing to do */
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START();
+
+	/*
+	 * We are expecting there to be zero FdwXact that need to be copied to
+	 * disk, so we perform all I/O while holding FdwXactLock for simplicity.
+	 * This presents any new foreign xacts from preparing while this occurs,
+	 * which shouldn't be a problem since the presence of long-lived prepared
+	 * foreign xacts indicated the transaction manager isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXact with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked invalid,
+	 * because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_fdwxacts; cnt++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+								fdwxact->serverid, fdwxact->userid,
+								buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.  FdwXact files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE();
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+				(errmsg_plural("%u foreign transaction state file was written "
+							   "for long-running prepared transactions",
+							   "%u foreign transaction state files were written "
+							   "for long-running prepared transactions",
+							   serialized_fdwxacts,
+							   serialized_fdwxacts)));
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
+									XL_ROUTINE(.page_read = &read_local_xlog_page,
+											   .segment_open = &wal_segment_open,
+											   .segment_close = &wal_segment_close),
+									NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+				 errdetail("Failed while allocating an XLog reading processor.")));
+
+	XLogBeginRead(xlogreader, lsn);
+	record = XLogReadRecord(xlogreader, &errormsg);
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not read foreign transaction state from xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						(uint32) (lsn >> 32),
+						(uint32) lsn)));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(Oid dbid, TransactionId xid, Oid serverid,
+					Oid userid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not recreate foreign transaction state file \"%s\": %m",
+						path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(Oid dbid, TransactionId xid, Oid serverid,
+					 Oid userid, XLogRecPtr insert_start_lsn, bool fromdisk)
+{
+	TransactionId origNextXid =
+	XidFromFullTransactionId(ShmemVariableCache->nextFullXid);
+	char	   *buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			RemoveFdwXactFile(dbid, xid, serverid, userid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u, server %u and user %u",
+							xid, serverid, userid)));
+			FdwXactRedoRemove(dbid, xid, serverid, userid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactFile(dbid, xid, serverid, userid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open FDW transaction state file \"%s\": %m",
+						path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactOnDiskData, fdwxact_id) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactOnDiskData *) buf;
+	if (fdwxact_file_data->dbid != dbid ||
+		fdwxact_file_data->serverid != serverid ||
+		fdwxact_file_data->userid != userid ||
+		fdwxact_file_data->local_xid != xid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.  ShmemVariableCache->nextFullXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.  Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextFullXid = ShmemVariableCache->nextFullXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextFullXid);
+	TransactionId result = origNextXid;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->local_xid, result))
+			result = fdwxact->local_xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXact depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXact files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+restoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId local_xid;
+			Oid			dbid;
+			Oid			serverid;
+			Oid			userid;
+			char	   *buf;
+
+			sscanf(clde->d_name, "%08x_%08x_%08x_%08x",
+				   &dbid, &local_xid, &serverid, &userid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(dbid, local_xid, serverid, userid,
+									   InvalidXLogRecPtr, true);
+
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactFile(Oid dbid, TransactionId xid, Oid serverid, Oid userid,
+				  bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactFilePath(path, dbid, xid, serverid, userid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Scan the shared memory entries of FdwXact and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->dbid, fdwxact->local_xid,
+								   fdwxact->serverid, fdwxact->userid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign prepared transaction %u for server %u and user %u from shared memory",
+						fdwxact->local_xid, fdwxact->serverid, fdwxact->userid)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+
+		/*
+		 * If the foreign transaction is part of the prepared local
+		 * transaction, it's not in in-doubt. The future COMMIT/ROLLBACK
+		 * PREPARED can determine the fate of this foreign transaction.
+		 */
+		if (TwoPhaseExists(fdwxact->local_xid))
+		{
+			ereport(DEBUG2,
+					(errmsg("clear in-doubt flag from foreign transaction %u, server %u, user %u as found the corresponding local prepared transaction",
+							fdwxact->local_xid, fdwxact->serverid,
+							fdwxact->userid)));
+			fdwxact->indoubt = false;
+		}
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXact		fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+}			WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	6
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		FdwXactStatus status;
+		bool		indoubt;
+		char	   *xact_status;
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+
+		if (!fdwxact->valid)
+			continue;
+
+		memset(nulls, 0, sizeof(nulls));
+
+		SpinLockAcquire(&fdwxact->mutex);
+		status = fdwxact->status;
+		indoubt = fdwxact->indoubt;
+		SpinLockRelease(&fdwxact->mutex);
+
+		values[0] = TransactionIdGetDatum(fdwxact->local_xid);
+		values[1] = ObjectIdGetDatum(fdwxact->serverid);
+		values[2] = ObjectIdGetDatum(fdwxact->userid);
+
+		switch (status)
+		{
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+		values[3] = CStringGetTextDatum(xact_status);
+		values[4] = BoolGetDatum(indoubt);
+		values[5] = PointerGetDatum(cstring_to_text_with_len(fdwxact->fdwxact_id,
+															 strlen(fdwxact->fdwxact_id)));
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
+
+/*
+ * Built-in SQL function to resolve a prepared foreign transaction.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			serverid = PG_GETARG_OID(1);
+	Oid			userid = PG_GETARG_OID(2);
+	int			idx;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to resolve foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	/* Find and hold the FdwXact entry */
+	idx = get_fdwxact(MyDatabaseId, xid, serverid, userid, true);
+
+	LWLockRelease(FdwXactLock);
+
+	if (idx < 0)
+	{
+		/* No entry */
+		PG_RETURN_BOOL(false);
+	}
+
+	PG_TRY();
+	{
+		FdwXactResolveFdwXacts(&idx, 1, NULL);
+	}
+	PG_CATCH();
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[idx];
+
+		SpinLockAcquire(&fdwxact->mutex);
+		FdwXactCtl->fdwxacts[idx]->held_by = InvalidBackendId;
+		SpinLockRelease(&fdwxact->mutex);
+		PG_RE_THROW();
+	}
+	PG_END_TRY();
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			serverid = PG_GETARG_OID(1);
+	Oid			userid = PG_GETARG_OID(2);
+	FdwXact		fdwxact;
+	int			i;
+
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 (errmsg("must be superuser to remove foreign transactions"))));
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->valid && fdwxact->dbid == MyDatabaseId &&
+			fdwxact->local_xid == xid && fdwxact->serverid == serverid &&
+			fdwxact->userid == userid)
+			break;
+	}
+
+	if (i >= FdwXactCtl->num_fdwxacts)
+	{
+		LWLockRelease(FdwXactLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("does not exist foreign transaction on server %u",
+						serverid)));
+	}
+
+	if (fdwxact->inprocessing || fdwxact->held_by != InvalidBackendId)
+	{
+		LWLockRelease(FdwXactLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot remove foreign transaction entry which is being processed")));
+	}
+
+	/* Clean up entry and any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactFile(fdwxact->dbid, fdwxact->local_xid,
+						  fdwxact->serverid, fdwxact->userid,
+						  true);
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/launcher.c b/src/backend/access/fdwxact/launcher.c
new file mode 100644
index 0000000000..fed2fbcd08
--- /dev/null
+++ b/src/backend/access/fdwxact/launcher.c
@@ -0,0 +1,560 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when arrived a requested by backend process.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "commands/dbcommands.h"
+#include "nodes/pg_list.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+static void fdwxact_launcher_onexit(int code, Datum arg);
+static void fdwxact_launcher_sighup(SIGNAL_ARGS);
+static void fdwxact_launch_resolver(Oid dbid);
+static bool fdwxact_relaunch_resolvers(void);
+
+static volatile sig_atomic_t got_SIGHUP = false;
+static volatile sig_atomic_t got_SIGUSR2 = false;
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+FdwXactLauncherRequestToLaunch(void)
+{
+	if (FdwXactRslvCtl->launcher_pid != InvalidPid)
+		kill(FdwXactRslvCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactRsoverShmemInit */
+Size
+FdwXactRslvShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactRslvCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactRslvShmemInit(void)
+{
+	bool		found;
+
+	FdwXactRslvCtl = ShmemInitStruct("Foreign transactions resolvers",
+									 FdwXactRslvShmemSize(),
+									 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int			slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactRslvCtl, 0, FdwXactRslvShmemSize());
+		SHMQueueInit(&(FdwXactRslvCtl->fdwxact_queue));
+		FdwXactRslvCtl->launcher_pid = InvalidPid;
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[slot];
+
+			memset(resolver, 0, sizeof(FdwXactResolver));
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+fdwxact_launcher_onexit(int code, Datum arg)
+{
+	FdwXactRslvCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGHUP: set flag to reload configuration at next convenient time */
+static void
+fdwxact_launcher_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+fdwxact_launcher_sigusr2(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(fdwxact_launcher_onexit, (Datum) 0);
+
+	Assert(FdwXactRslvCtl->launcher_pid == InvalidPid);
+	FdwXactRslvCtl->launcher_pid = MyProcPid;
+	FdwXactRslvCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, fdwxact_launcher_sighup);
+	pqsignal(SIGUSR2, fdwxact_launcher_sigusr2);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz now;
+		long		wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int			rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a
+		 * foreign_xact_resolution_retry_interval but always starts when the
+		 * backend requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool		launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			/*
+			 * Launch foreign transaction resolvers that are requested but not
+			 * running.
+			 */
+			launched = fdwxact_relaunch_resolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we should
+			 * retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request launcher to launch a new foreign transaction resolver process
+ * or wake up the resolver if it's already running.
+ */
+void
+FdwXactLaunchOrWakeupResolver(void)
+{
+	volatile FdwXactResolver *resolver;
+	bool		found = false;
+	int			i;
+
+	/*
+	 * Looking for a resolver process that is running and working on the same
+	 * database.
+	 */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId)
+		{
+			found = true;
+			break;
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (found)
+	{
+		/* Found the running resolver */
+		elog(DEBUG1,
+			 "found a running foreign transaction resolver process for database %u",
+			 MyDatabaseId);
+
+		/*
+		 * Wakeup the resolver. It's possible that the resolver is starting up
+		 * and doesn't attach its slot yet. Since the resolver will find
+		 * FdwXact entry we inserted soon we don't anything.
+		 */
+		if (resolver->latch)
+			SetLatch(resolver->latch);
+
+		return;
+	}
+
+	/* Otherwise wake up the launcher to launch new resolver */
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid'.
+ */
+static void
+fdwxact_launch_resolver(Oid dbid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int			unused_slot = -1;
+	int			i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign transaction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactRslvCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for database %u", resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to
+	 * wait until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Launch or relaunch foreign transaction resolvers on database that has
+ * at least one FdwXact entry but no resolvers are running on it.
+ */
+static bool
+fdwxact_relaunch_resolvers(void)
+{
+	HTAB	   *resolver_dbs;	/* DBs resolver's running on */
+	HTAB	   *fdwxact_dbs;	/* DBs having at least one FdwXact entry */
+	HASHCTL		ctl;
+	HASH_SEQ_STATUS status;
+	Oid		   *entry;
+	bool		launched;
+	int			i;
+
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(Oid);
+	ctl.entrysize = sizeof(Oid);
+	fdwxact_dbs = hash_create("fdwxact dblist",
+							  32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids that has at least one non-in-doubt FdwXact entry */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+		bool		indoubt;
+		BackendId	held_by;
+
+		if (!fdwxact->valid)
+			continue;
+
+		SpinLockAcquire(&fdwxact->mutex);
+		indoubt = fdwxact->indoubt;
+		held_by = fdwxact->held_by;
+		SpinLockRelease(&fdwxact->mutex);
+
+		if ((indoubt && held_by == InvalidBackendId) ||
+			(!indoubt && held_by != InvalidBackendId))
+			hash_search(fdwxact_dbs, &(fdwxact->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no FdwXact entry, no need to launch new one */
+	if (hash_get_num_entries(fdwxact_dbs) == 0)
+	{
+		hash_destroy(fdwxact_dbs);
+		return false;
+	}
+
+	resolver_dbs = hash_create("resolver dblist",
+							   32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	/* Collect database oids on which resolvers are running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactRslvCtl->resolvers[i];
+
+		if (!resolver->in_use)
+			continue;
+
+		hash_search(resolver_dbs, &(resolver->dbid), HASH_ENTER, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Find DBs on which no resolvers are running and launch new one on them */
+	hash_seq_init(&status, fdwxact_dbs);
+	while ((entry = (Oid *) hash_seq_search(&status)) != NULL)
+	{
+		bool		found;
+
+		hash_search(resolver_dbs, entry, HASH_FIND, &found);
+
+		if (!found)
+		{
+			/* No resolver is running on this database, launch new one */
+			fdwxact_launch_resolver(*entry);
+			launched = true;
+		}
+	}
+
+	hash_destroy(fdwxact_dbs);
+	hash_destroy(resolver_dbs);
+
+	return launched;
+}
+
+/* Register a background worker running the foreign transaction launcher */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactRslvCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Stop the fdwxact resolver running on the given database.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	Oid			dbid = PG_GETARG_OID(0);
+	FdwXactResolver *resolver = NULL;
+	int			i;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	if (!OidIsValid(dbid))
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid database id")));
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	/* Find the running resolver process on the given database */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		resolver = &FdwXactRslvCtl->resolvers[i];
+
+		/* found! */
+		if (resolver->in_use && resolver->dbid == dbid)
+			break;
+	}
+
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errmsg("there is no running foreign transaction resolver process on database %d",
+						dbid)));
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int			rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		/* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+					   10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/fdwxact/resolver.c b/src/backend/access/fdwxact/resolver.c
new file mode 100644
index 0000000000..b91a2e1e88
--- /dev/null
+++ b/src/backend/access/fdwxact/resolver.c
@@ -0,0 +1,436 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves foreign
+ * transactions that participate to a distributed transaction. A resolver
+ * process is started by foreign transaction launcher for each databases.
+ *
+ * A resolver process continues to resolve foreign transactions on the
+ * database, which the backend process is waiting for resolution.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency  termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/fdwxact/resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int			foreign_xact_resolution_retry_interval;
+int			foreign_xact_resolver_timeout = 60 * 1000;
+
+FdwXactRslvCtlData *FdwXactRslvCtl;
+
+static void FXRslvLoop(void);
+static long FXRslvComputeSleepTime(TimestampTz now, TimestampTz targetTime);
+static void FXRslvCheckTimeout(TimestampTz now);
+
+static void fdwxact_resolver_sighup(SIGNAL_ARGS);
+static void fdwxact_resolver_onexit(int code, Datum arg);
+static void fdwxact_resolver_detach(void);
+static void fdwxact_resolver_attach(int slot);
+static void hold_fdwxacts(PGPROC *waiter);
+static void hold_indoubt_fdwxacts(void);
+
+/* Flags set by signal handlers */
+static volatile sig_atomic_t got_SIGHUP = false;
+static TimestampTz last_resolution_time = -1;
+
+/*
+ * held_fdwxacts has indexes of FdwXact which the resolver marked
+ * as in-processing. We clear that flag from those entries on failure.
+ */
+static int *held_fdwxacts = NULL;
+static int	nheld;
+
+/* Set flag to reload configuration at next convenient time */
+static void
+fdwxact_resolver_sighup(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGHUP = true;
+
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+fdwxact_resolver_detach(void)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info.
+ */
+static void
+fdwxact_resolver_onexit(int code, Datum arg)
+{
+	fdwxact_resolver_detach();
+
+	/* clear inprocessing flags */
+	for (int i = 0; i < nheld; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[held_fdwxacts[i]];
+
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwxact->inprocessing = false;
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	FdwXactLauncherRequestToLaunch();
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+fdwxact_resolver_attach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactRslvCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+
+	before_shmem_exit(fdwxact_resolver_onexit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int			slot = DatumGetInt32(main_arg);
+
+	/* Attach to a slot */
+	fdwxact_resolver_attach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, fdwxact_resolver_sighup);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for database \"%s\" has started",
+					get_database_name(MyFdwXactResolver->dbid))));
+	CommitTransactionCommand();
+
+	held_fdwxacts = palloc(sizeof(int) * max_prepared_foreign_xacts);
+	nheld = 0;
+
+	/* Initialize stats to a sanish value */
+	last_resolution_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FXRslvLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FXRslvLoop(void)
+{
+	MemoryContext resolver_ctx;
+
+	resolver_ctx = AllocSetContextCreate(TopMemoryContext,
+										 "Foreign Transaction Resolver",
+										 ALLOCSET_DEFAULT_SIZES);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TransactionId waitXid = InvalidTransactionId;
+		TimestampTz resolutionTs = -1;
+		TimestampTz now;
+		int			rc;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		MemoryContextSwitchTo(resolver_ctx);
+
+		if (got_SIGHUP)
+		{
+			got_SIGHUP = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Process waiter until either the queue gets empty or the queue has
+		 * only waiters that have a future resolution timestamp.
+		 */
+		for (;;)
+		{
+			PGPROC	   *waiter;
+
+			CHECK_FOR_INTERRUPTS();
+
+			LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+
+			waiter = FdwXactGetWaiter(now, &resolutionTs, &waitXid);
+
+			if (!waiter)
+			{
+				/* Not found, break */
+				LWLockRelease(FdwXactResolutionLock);
+				break;
+			}
+
+			/* Hold the waiting foreign transactions */
+			hold_fdwxacts(waiter);
+			Assert(nheld > 0);
+			LWLockRelease(FdwXactResolutionLock);
+
+			/* Resolve the waiting distributed transaction */
+			StartTransactionCommand();
+			FdwXactResolveFdwXacts(held_fdwxacts, nheld, waiter);
+			CommitTransactionCommand();
+
+			last_resolution_time = now;
+		}
+
+		/* Hold in-doubt transactions */
+		hold_indoubt_fdwxacts();
+
+		if (nheld > 0)
+		{
+			StartTransactionCommand();
+			FdwXactResolveFdwXacts(held_fdwxacts, nheld, NULL);
+			CommitTransactionCommand();
+		}
+
+		FXRslvCheckTimeout(now);
+
+		sleep_time = FXRslvComputeSleepTime(now, resolutionTs);
+
+		MemoryContextResetAndDeleteChildren(resolver_ctx);
+		MemoryContextSwitchTo(TopMemoryContext);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FXRslvCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(last_resolution_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	LWLockAcquire(FdwXactResolutionLock, LW_SHARED);
+	if (!FdwXactWaiterExists(MyDatabaseId))
+	{
+		/* There is no waiting backend */
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("foreign transaction resolver for database \"%s\" will stop because the timeout",
+						get_database_name(MyDatabaseId))));
+		CommitTransactionCommand();
+
+		/*
+		 * Keep holding FdwXactResolutionLock until detached the slot. It is
+		 * necessary to prevent a race condition; a waiter enqueues after
+		 * FdwXactWaiterExists check.
+		 */
+		fdwxact_resolver_detach();
+		LWLockRelease(FdwXactResolutionLock);
+		proc_exit(0);
+	}
+	else
+		elog(DEBUG2, "resolver reached to the timeout but don't exist as the queue is not empty");
+
+	LWLockRelease(FdwXactResolutionLock);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out or the next resolution time given by nextResolutionTs.
+ */
+static long
+FXRslvComputeSleepTime(TimestampTz now, TimestampTz nextResolutionTs)
+{
+	long		sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(last_resolution_time,
+											  foreign_xact_resolver_timeout);
+		TimestampDifference(now, timeout,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	if (nextResolutionTs > 0)
+	{
+		long		sec_to_timeout;
+		int			microsec_to_timeout;
+
+		TimestampDifference(now, nextResolutionTs,
+							&sec_to_timeout, &microsec_to_timeout);
+
+		sleeptime = Min(sleeptime,
+						sec_to_timeout * 1000 + microsec_to_timeout / 1000);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
+
+/*
+ * Mark in-doubt transactions as in-processing.
+ */
+static void
+hold_indoubt_fdwxacts(void)
+{
+	nheld = 0;
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->valid && fdwxact->dbid == MyDatabaseId &&
+			fdwxact->held_by == InvalidBackendId && fdwxact->indoubt)
+		{
+			held_fdwxacts[nheld++] = i;
+
+			/* hold lock */
+			SpinLockAcquire(&fdwxact->mutex);
+			fdwxact->inprocessing = true;
+			SpinLockRelease(&fdwxact->mutex);
+		}
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Mark foreign transactions associated with the given waiter's transaction
+ * as in-processing.
+ */
+static void
+hold_fdwxacts(PGPROC *waiter)
+{
+	nheld = 0;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_fdwxacts; i++)
+	{
+		FdwXact		fdwxact = FdwXactCtl->fdwxacts[i];
+
+		if (fdwxact->valid && fdwxact->dbid == waiter->databaseId &&
+			fdwxact->local_xid == waiter->fdwXactWaitXid)
+		{
+			held_fdwxacts[nheld++] = i;
+
+			/* hold lock */
+			SpinLockAcquire(&fdwxact->mutex);
+			Assert(!fdwxact->indoubt);
+			Assert(fdwxact->held_by = waiter->backendId);
+			fdwxact->inprocessing = true;
+			SpinLockRelease(&fdwxact->mutex);
+		}
+	}
+	LWLockRelease(FdwXactLock);
+}
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index f88d72fd86..982c1a36cc 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -13,6 +13,7 @@ OBJS = \
 	clogdesc.o \
 	committsdesc.o \
 	dbasedesc.o \
+	fdwxactdesc.o \
 	genericdesc.o \
 	gindesc.o \
 	gistdesc.o \
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000000..ca761763e5
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,58 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxactdesc.c
+ *		PostgreSQL global transaction manager for foreign server.
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdwxactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdwxact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		FdwXactOnDiskData *fdwxact_insert = (FdwXactOnDiskData *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_insert->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_insert->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_insert->dbid);
+		appendStringInfo(buf, " local xid: %u,", fdwxact_insert->local_xid);
+		appendStringInfo(buf, " id: %s", fdwxact_insert->fdwxact_id);
+	}
+	else
+	{
+		xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec;
+
+		appendStringInfo(buf, "server: %u,", fdwxact_remove->serverid);
+		appendStringInfo(buf, " user: %u,", fdwxact_remove->userid);
+		appendStringInfo(buf, " database: %u,", fdwxact_remove->dbid);
+		appendStringInfo(buf, " local xid: %u", fdwxact_remove->xid);
+	}
+
+}
+
+const char *
+fdwxact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDWXACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDWXACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index 1cd97852e8..ea045174e0 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -114,7 +114,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s "
+						 "max_prepared_foreign_transactions=%d",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
@@ -122,7 +123,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
-						 xlrec.track_commit_timestamp ? "on" : "off");
+						 xlrec.track_commit_timestamp ? "on" : "off",
+						 xlrec.max_prepared_foreign_xacts);
 	}
 	else if (info == XLOG_FPW_CHANGE)
 	{
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 58091f6b52..200cf9d067 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -10,6 +10,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index e1904877fa..2b9e039580 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -850,6 +851,35 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 	return result;
 }
 
+/*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+		PGXACT	*pgxact = &ProcGlobal->allPgXact[gxact->pgprocno];
+
+		if (pgxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
 /*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
@@ -2196,6 +2226,9 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	XLogRecPtr	recptr;
 	TimestampTz committs = GetCurrentTimestamp();
 	bool		replorigin;
+	bool		need_fdwxact_commit;
+
+	need_fdwxact_commit = PrepareFdwXactParticipants(xid);
 
 	/*
 	 * Are we using the replication origins feature?  Or, in other words, are
@@ -2266,6 +2299,16 @@ RecordTransactionCommitPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, true);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be committed.
+	 */
+	if (need_fdwxact_commit)
+	{
+		SetFdwXactParticipants(xid, true);
+		FdwXactWaitForResolution(xid);
+	}
 }
 
 /*
@@ -2285,6 +2328,9 @@ RecordTransactionAbortPrepared(TransactionId xid,
 							   const char *gid)
 {
 	XLogRecPtr	recptr;
+	bool		need_fdwxact_commit;
+
+	need_fdwxact_commit = PrepareFdwXactParticipants(xid);
 
 	/*
 	 * Catch the scenario where we aborted partway through
@@ -2325,6 +2371,16 @@ RecordTransactionAbortPrepared(TransactionId xid,
 	 * in the procarray and continue to hold locks.
 	 */
 	SyncRepWaitForLSN(recptr, false);
+
+	/*
+	 * Wait for foreign transaction prepared as part of this prepared
+	 * transaction to be rolled back.
+	 */
+	if (need_fdwxact_commit)
+	{
+		SetFdwXactParticipants(xid, false);
+		FdwXactWaitForResolution(xid);
+	}
 }
 
 /*
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index cd30b62d36..c611fd8b45 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
 #include "access/subtrans.h"
@@ -1219,6 +1220,7 @@ RecordTransactionCommit(void)
 	SharedInvalidationMessage *invalMessages = NULL;
 	bool		RelcacheInitFileInval = false;
 	bool		wrote_xlog;
+	bool		need_commit_globally;
 
 	/* Get data needed for commit record */
 	nrels = smgrGetPendingDeletes(true, &rels);
@@ -1227,6 +1229,7 @@ RecordTransactionCommit(void)
 		nmsgs = xactGetCommittedInvalidationMessages(&invalMessages,
 													 &RelcacheInitFileInval);
 	wrote_xlog = (XactLastRecEnd != 0);
+	need_commit_globally = FdwXactIsForeignTwophaseCommitRequired();
 
 	/*
 	 * If we haven't been assigned an XID yet, we neither can, nor do we want
@@ -1265,12 +1268,13 @@ RecordTransactionCommit(void)
 		}
 
 		/*
-		 * If we didn't create XLOG entries, we're done here; otherwise we
-		 * should trigger flushing those entries the same as a commit record
+		 * If we didn't create XLOG entries and the transaction does not need
+		 * to be committed using two-phase commit. we're done here; otherwise
+		 * we should trigger flushing those entries the same as a commit record
 		 * would.  This will primarily happen for HOT pruning and the like; we
 		 * want these to be flushed to disk in due time.
 		 */
-		if (!wrote_xlog)
+		if (!wrote_xlog && !need_commit_globally)
 			goto cleanup;
 	}
 	else
@@ -1428,6 +1432,14 @@ RecordTransactionCommit(void)
 	if (wrote_xlog && markXidCommitted)
 		SyncRepWaitForLSN(XactLastRecEnd, true);
 
+	/*
+	 * Wait for prepared foreign transaction to be resolved, if required.
+	 * We only want to wait if we prepared foreign transaction in this
+	 * transaction.
+	 */
+	if (need_commit_globally && markXidCommitted)
+		FdwXactWaitForResolution(xid);
+
 	/* remember end of last commit record */
 	XactLastCommitEnd = XactLastRecEnd;
 
@@ -2087,6 +2099,9 @@ CommitTransaction(void)
 			break;
 	}
 
+	/* Pre-commit step for foreign transactions */
+	PreCommit_FdwXact();
+
 	CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT
 					  : XACT_EVENT_PRE_COMMIT);
 
@@ -2254,6 +2269,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXact(true);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2341,6 +2357,8 @@ PrepareTransaction(void)
 	 * the transaction-abort path.
 	 */
 
+	AtPrepare_FdwXact();
+
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
 
@@ -2532,6 +2550,9 @@ PrepareTransaction(void)
 	 */
 	PostPrepare_Twophase();
 
+	/* Release held FdwXact entries */
+	PostPrepare_FdwXact();
+
 	/* PREPARE acts the same as COMMIT as far as GUC is concerned */
 	AtEOXact_GUC(true, 1);
 	AtEOXact_SPI(true);
@@ -2542,6 +2563,7 @@ PrepareTransaction(void)
 	AtEOXact_Files(true);
 	AtEOXact_ComboCid();
 	AtEOXact_HashTables(true);
+	//AtEOXact_FdwXact(true);
 	/* don't call AtEOXact_PgStat here; we fixed pgstat state above */
 	AtEOXact_Snapshot(true, true);
 	pgstat_report_xact_timestamp(0);
@@ -2751,6 +2773,7 @@ AbortTransaction(void)
 		AtEOXact_HashTables(false);
 		AtEOXact_PgStat(false, is_parallel_worker);
 		AtEOXact_ApplyLauncher(false);
+		AtEOXact_FdwXact(false);
 		pgstat_report_xact_timestamp(0);
 	}
 
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index ca09d81b08..eae8c60db3 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
@@ -4599,6 +4600,7 @@ InitControlFile(uint64 sysidentifier)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6286,6 +6288,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_wal_senders",
 									 max_wal_senders,
 									 ControlFile->max_wal_senders);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
@@ -6836,14 +6841,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	restoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -7045,7 +7051,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7558,6 +7567,7 @@ StartupXLOG(void)
 	 * as potential problems are detected before any on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Consider whether we need to assign a new timeline ID.
@@ -7888,6 +7898,9 @@ StartupXLOG(void)
 	/* Reload shared-memory state for prepared transactions */
 	RecoverPreparedTransactions();
 
+	/* Load all foreign transaction entries from disk to memory */
+	RecoverFdwXacts();
+
 	/*
 	 * Shutdown the recovery environment. This must occur after
 	 * RecoverPreparedTransactions(), see notes for lock_twophase_recover()
@@ -9183,6 +9196,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointReplicationOrigin();
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9712,8 +9726,10 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
-		track_commit_timestamp != ControlFile->track_commit_timestamp)
+		track_commit_timestamp != ControlFile->track_commit_timestamp ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts)
 	{
 		/*
 		 * The change in number of backend slots doesn't need to be WAL-logged
@@ -9731,6 +9747,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9747,6 +9764,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -9952,6 +9970,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -10151,6 +10170,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 56420bbc9d..56af9e6408 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -333,6 +333,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+       SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
index 6d53dc463c..a1dea253c2 100644
--- a/src/backend/commands/copy.c
+++ b/src/backend/commands/copy.c
@@ -2807,8 +2807,14 @@ CopyFrom(CopyState cstate)
 
 	if (resultRelInfo->ri_FdwRoutine != NULL &&
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(resultRelInfo->ri_RelationDesc),
+							   true);
+
 		resultRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate,
 														 resultRelInfo);
+	}
 
 	/* Prepare to catch AFTER triggers. */
 	AfterTriggerBeginQuery();
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index a399ab4de9..8f2f7041e8 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1101,6 +1103,18 @@ RemoveForeignServerById(Oid srvId)
 	if (!HeapTupleIsValid(tp))
 		elog(ERROR, "cache lookup failed for foreign server %u", srvId);
 
+	/*
+	 * If there is a foreign prepared transaction with this foreign server,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (FdwXactExists(MyDatabaseId, srvId, InvalidOid))
+	{
+		Form_pg_foreign_server srvForm = (Form_pg_foreign_server) GETSTRUCT(tp);
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transactions on it",
+						NameStr(srvForm->srvname))));
+	}
+
 	CatalogTupleDelete(rel, &tp->t_self);
 
 	ReleaseSysCache(tp);
@@ -1422,6 +1436,15 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
+	/*
+	 * If there is a foreign prepared transaction with this user mapping,
+	 * dropping it might result in dangling prepared transaction.
+	 */
+	if (FdwXactExists(MyDatabaseId, srv->serverid,	useId))
+		ereport(WARNING,
+				(errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"",
+						srv->servername, MappingUserName(useId))));
+
 	/*
 	 * Do the deletion
 	 */
@@ -1575,6 +1598,13 @@ ImportForeignSchema(ImportForeignSchemaStmt *stmt)
 				 errmsg("foreign-data wrapper \"%s\" does not support IMPORT FOREIGN SCHEMA",
 						fdw->fdwname)));
 
+	/*
+	 * Remember the transaction accesses to a foreign server. Normally during
+	 * ImportForeignSchema we don't modify data on foreign servers, so remember it
+	 * as not-modified server.
+	 */
+	RegisterFdwXactByServerId(server->serverid, false);
+
 	/* Call FDW to get a list of commands */
 	cmd_list = fdw_routine->ImportForeignSchema(stmt, server->serverid);
 
diff --git a/src/backend/executor/execPartition.c b/src/backend/executor/execPartition.c
index fb6ce49056..3fa8bfe09f 100644
--- a/src/backend/executor/execPartition.c
+++ b/src/backend/executor/execPartition.c
@@ -13,6 +13,7 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/table.h"
 #include "access/tableam.h"
 #include "catalog/partition.h"
@@ -939,7 +940,14 @@ ExecInitRoutingInfo(ModifyTableState *mtstate,
 	 */
 	if (partRelInfo->ri_FdwRoutine != NULL &&
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert != NULL)
+	{
+		Relation		child = partRelInfo->ri_RelationDesc;
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(RelationGetRelid(child), true);
+
 		partRelInfo->ri_FdwRoutine->BeginForeignInsert(mtstate, partRelInfo);
+	}
 
 	partRelInfo->ri_PartitionInfo = partrouteinfo;
 	partRelInfo->ri_CopyMultiInsertBuffer = NULL;
diff --git a/src/backend/executor/nodeForeignscan.c b/src/backend/executor/nodeForeignscan.c
index 513471ab9b..29f376e48c 100644
--- a/src/backend/executor/nodeForeignscan.c
+++ b/src/backend/executor/nodeForeignscan.c
@@ -22,6 +22,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/xact.h"
 #include "executor/executor.h"
 #include "executor/nodeForeignscan.h"
 #include "foreign/fdwapi.h"
@@ -224,9 +226,31 @@ ExecInitForeignScan(ForeignScan *node, EState *estate, int eflags)
 	 * Tell the FDW to initialize the scan.
 	 */
 	if (node->operation != CMD_SELECT)
+	{
+		RangeTblEntry	*rte;
+
+		rte = exec_rt_fetch(estate->es_result_relation_info->ri_RangeTableIndex,
+							estate);
+
+		/* Remember the transaction modifies data on a foreign server*/
+		RegisterFdwXactByRelId(rte->relid, true);
+
 		fdwroutine->BeginDirectModify(scanstate, eflags);
+	}
 	else
+	{
+		RangeTblEntry	*rte;
+		int rtindex = (scanrelid > 0) ?
+			scanrelid :
+			bms_next_member(node->fs_relids, -1);
+
+		rte = exec_rt_fetch(rtindex, estate);
+
+		/* Remember the transaction accesses to a foreign server */
+		RegisterFdwXactByRelId(rte->relid, false);
+
 		fdwroutine->BeginForeignScan(scanstate, eflags);
+	}
 
 	return scanstate;
 }
diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 1ec07bad07..e5dee94764 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -37,6 +37,7 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
 #include "access/tableam.h"
@@ -47,6 +48,7 @@
 #include "executor/executor.h"
 #include "executor/nodeModifyTable.h"
 #include "foreign/fdwapi.h"
+#include "foreign/foreign.h"
 #include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "rewrite/rewriteHandler.h"
@@ -2418,6 +2420,10 @@ ExecInitModifyTable(ModifyTable *node, EState *estate, int eflags)
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify != NULL)
 		{
 			List	   *fdw_private = (List *) list_nth(node->fdwPrivLists, i);
+			Oid			relid = RelationGetRelid(resultRelInfo->ri_RelationDesc);
+
+			/* Remember the transaction modifies data on a foreign server*/
+			RegisterFdwXactByRelId(relid, true);
 
 			resultRelInfo->ri_FdwRoutine->BeginForeignModify(mtstate,
 															 resultRelInfo,
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index 61e48ca3f8..8f411c0559 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -187,6 +187,49 @@ GetForeignServerByName(const char *srvname, bool missing_ok)
 	return GetForeignServer(serverid);
 }
 
+/*
+ * GetUserMappingOid - look up the user mapping by user mapping oid.
+ *
+ * If userid of the mapping is invalid, we set it to current userid.
+ */
+UserMapping *
+GetUserMappingByOid(Oid umid)
+{
+	Datum		datum;
+	HeapTuple   tp;
+	UserMapping	*um;
+	bool		isnull;
+	Form_pg_user_mapping tableform;
+
+	tp = SearchSysCache1(USERMAPPINGOID,
+						 ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		ereport(ERROR,
+				(errcode(ERRCODE_UNDEFINED_OBJECT),
+				 errmsg("user mapping not found for %d", umid)));
+
+	tableform = (Form_pg_user_mapping) GETSTRUCT(tp);
+	um = (UserMapping *) palloc(sizeof(UserMapping));
+	um->umid = umid;
+	um->userid = OidIsValid(tableform->umuser) ?
+		tableform->umuser : GetUserId();
+	um->serverid = tableform->umserver;
+
+	/* Extract the umoptions */
+	datum = SysCacheGetAttr(USERMAPPINGUSERSERVER,
+							tp,
+							Anum_pg_user_mapping_umoptions,
+							&isnull);
+	if (isnull)
+		um->options = NIL;
+	else
+		um->options = untransformRelOptions(datum);
+
+	ReleaseSysCache(tp);
+
+	return um;
+}
 
 /*
  * GetUserMapping - look up the user mapping.
@@ -328,6 +371,18 @@ GetFdwRoutine(Oid fdwhandler)
 		elog(ERROR, "foreign-data wrapper handler function %u did not return an FdwRoutine struct",
 			 fdwhandler);
 
+	/* Sanity check for transaction management callbacks */
+	if ((routine->CommitForeignTransaction && !routine->RollbackForeignTransaction) ||
+		(!routine->CommitForeignTransaction && routine->RollbackForeignTransaction))
+		elog(ERROR,
+			 "foreign-data wrapper must support both commit and rollback routines or neither");
+
+	if (routine->PrepareForeignTransaction &&
+		!routine->CommitForeignTransaction &&
+		!routine->RollbackForeignTransaction)
+		elog(ERROR,
+			 "foreign-data wrapper that supports prepare routine must support both commit and rollback routines");
+
 	return routine;
 }
 
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index beb5e85434..2258424e81 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -12,6 +12,8 @@
 
 #include "postgres.h"
 
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
@@ -128,6 +130,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index d7f99d9944..84bb1913f3 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -3667,6 +3667,12 @@ pgstat_get_wait_activity(WaitEventActivity w)
 		case WAIT_EVENT_CHECKPOINTER_MAIN:
 			event_name = "CheckpointerMain";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
 		case WAIT_EVENT_LOGICAL_APPLY_MAIN:
 			event_name = "LogicalApplyMain";
 			break;
@@ -3777,6 +3783,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_EXECUTE_GATHER:
 			event_name = "ExecuteGather";
 			break;
+		case WAIT_EVENT_FDWXACT_RESOLUTION:
+			event_name = "FdwXactResolution";
+			break;
 		case WAIT_EVENT_HASH_BATCH_ALLOCATE:
 			event_name = "HashBatchAllocate";
 			break;
@@ -4103,6 +4112,15 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_TWOPHASE_FILE_WRITE:
 			event_name = "TwophaseFileWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
 		case WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ:
 			event_name = "WALSenderTimelineHistoryRead";
 			break;
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 160afe9f39..6a83f19e24 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,8 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -909,6 +911,10 @@ PostmasterMain(int argc, char *argv[])
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
 
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers == 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
+
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
 	 * (Put any slow processing further down, after postmaster.pid creation.)
@@ -973,12 +979,13 @@ PostmasterMain(int argc, char *argv[])
 	LocalProcessControlFile(false);
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index c2e5e3abf8..9d34817f39 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -151,6 +151,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 427b0d59cd..55609eed81 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -147,6 +149,8 @@ CreateSharedMemoryAndSemaphores(void)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactRslvShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -263,6 +267,8 @@ CreateSharedMemoryAndSemaphores(void)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactRslvShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 3c2b369615..56c43cf741 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -94,6 +94,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allPgXact[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -249,6 +251,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 	}
 
 	allProcs = ProcGlobal->allProcs;
@@ -1311,6 +1314,7 @@ GetOldestXmin(Relation rel, int flags)
 
 	TransactionId replication_slot_xmin = InvalidTransactionId;
 	TransactionId replication_slot_catalog_xmin = InvalidTransactionId;
+	TransactionId fdwxact_unresolved_xmin = InvalidTransactionId;
 
 	/*
 	 * If we're not computing a relation specific limit, or if a shared
@@ -1376,6 +1380,7 @@ GetOldestXmin(Relation rel, int flags)
 	 */
 	replication_slot_xmin = procArray->replication_slot_xmin;
 	replication_slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	if (RecoveryInProgress())
 	{
@@ -1425,6 +1430,15 @@ GetOldestXmin(Relation rel, int flags)
 		NormalTransactionIdPrecedes(replication_slot_xmin, result))
 		result = replication_slot_xmin;
 
+	/*
+	 * Check whether there are unresolved distributed transaction
+	 * requiring an older xmin.
+	 */
+	if (!(flags & PROCARRAY_FDWXACT_XMIN) &&
+		TransactionIdIsValid(fdwxact_unresolved_xmin) &&
+		NormalTransactionIdPrecedes(fdwxact_unresolved_xmin, result))
+		result = fdwxact_unresolved_xmin;
+
 	/*
 	 * After locks have been released and vacuum_defer_cleanup_age has been
 	 * applied, check whether we need to back up further to make logical
@@ -3125,6 +3139,38 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations of the xmin horizon to prevent
+ * vacuum clog from affected transactions still needed by resolving
+ * distributed transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
+/*
+ * ProcArrayGetFdwXactUnresolvedXmin
+ *
+ * Return the current unresolved xmin limits.
+ */
+TransactionId
+ProcArrayGetFdwXactUnresolvedXmin(void)
+{
+	TransactionId xmin;
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	xmin = procArray->fdwxact_unresolved_xmin;
+	LWLockRelease(ProcArrayLock);
+
+	return xmin;
+}
 
 #define XidCacheRemove(i) \
 	do { \
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index e6985e8eed..241b099238 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -50,3 +50,6 @@ MultiXactTruncationLock				41
 OldSnapshotTimeMapLock				42
 LogicalRepWorkerLock				43
 XactTruncationLock					44
+FdwXactLock							45
+FdwXactResolverLock					46
+FdwXactResolutionLock				47
diff --git a/src/backend/storage/lmgr/proc.c b/src/backend/storage/lmgr/proc.c
index f5eef6fa4e..9bd1e1791a 100644
--- a/src/backend/storage/lmgr/proc.c
+++ b/src/backend/storage/lmgr/proc.c
@@ -35,6 +35,7 @@
 #include <unistd.h>
 #include <sys/time.h>
 
+#include "access/fdwxact.h"
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/xact.h"
@@ -421,6 +422,10 @@ InitProcess(void)
 	MyProc->syncRepState = SYNC_REP_NOT_WAITING;
 	SHMQueueElemInit(&(MyProc->syncRepLinks));
 
+	/* Initialize fields for fdw xact */
+	MyProc->fdwXactState = FDWXACT_NOT_WAITING;
+	SHMQueueElemInit(&(MyProc->fdwXactLinks));
+
 	/* Initialize fields for group XID clearing. */
 	MyProc->procArrayGroupMember = false;
 	MyProc->procArrayGroupMemberXid = InvalidTransactionId;
@@ -822,6 +827,9 @@ ProcKill(int code, Datum arg)
 	/* Make sure we're out of the sync rep lists */
 	SyncRepCleanupAtProcExit();
 
+	/* Make sure we're out of the fdwxact lists */
+	FdwXactCleanupAtProcExit();
+
 #ifdef USE_ASSERT_CHECKING
 	{
 		int			i;
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 8958ec8103..5ed6c05b18 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3054,6 +3056,18 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 2f3e0a70e0..313bf33324 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -27,6 +27,7 @@
 #endif
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -426,6 +427,24 @@ static const struct config_enum_entry synchronous_commit_options[] = {
 	{NULL, 0, false}
 };
 
+/*
+ * Although only "required" and "disabled" are documented, we accept all
+ * the likely variants of "on" and "off".
+ */
+static const struct config_enum_entry foreign_twophase_commit_options[] = {
+	{"required", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"disabled", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"on", FOREIGN_TWOPHASE_COMMIT_REQUIRED, false},
+	{"off", FOREIGN_TWOPHASE_COMMIT_DISABLED, false},
+	{"true", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"false", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"yes", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"no", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{"1", FOREIGN_TWOPHASE_COMMIT_REQUIRED, true},
+	{"0", FOREIGN_TWOPHASE_COMMIT_DISABLED, true},
+	{NULL, 0, false}
+};
+
 /*
  * Although only "on", "off", "try" are documented, we accept all the likely
  * variants of "on" and "off".
@@ -763,6 +782,10 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FOREIGN_TRANSACTION */
+	gettext_noop("Foreign Transaction"),
+	/* FOREIGN_TRANSACTION_RESOLVER */
+	gettext_noop("Foreign Transaction / Resolver"),
 	/* COMPAT_OPTIONS */
 	gettext_noop("Version and Platform Compatibility"),
 	/* COMPAT_OPTIONS_PREVIOUS */
@@ -2471,6 +2494,52 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	/*
+	 * See also CheckRequiredParameterValues() if this parameter changes
+	 */
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FOREIGN_TRANSACTION_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FOREIGN_TRANSACTION_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
@@ -4599,6 +4668,16 @@ static struct config_enum ConfigureNamesEnum[] =
 		NULL, assign_synchronous_commit, NULL
 	},
 
+	{
+		{"foreign_twophase_commit", PGC_USERSET, FOREIGN_TRANSACTION,
+		 gettext_noop("Use of foreign twophase commit for the current transaction."),
+			NULL
+		},
+		&foreign_twophase_commit,
+		FOREIGN_TWOPHASE_COMMIT_DISABLED, foreign_twophase_commit_options,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"archive_mode", PGC_POSTMASTER, WAL_ARCHIVING,
 			gettext_noop("Allows archiving of WAL files using archive_command."),
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index 81055edde7..03de9fbc81 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -125,6 +125,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -344,6 +346,20 @@
 #max_sync_workers_per_subscription = 2	# taken from max_logical_replication_workers
 
 
+#------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+#foreign_twophase_commit = disabled	# use two-phase commit for distributed transactions:
+					# disabled or required
+
 #------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index a0b0458108..8701c5f005 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -81,6 +81,8 @@ provider postgresql {
 	probe multixact__checkpoint__done(bool);
 	probe twophase__checkpoint__start();
 	probe twophase__checkpoint__done();
+	probe fdwxact__checkpoint__start();
+	probe fdwxact__checkpoint__done();
 
 	probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int);
 	probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int);
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 4ff0c6c700..b3f4f8fe18 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -208,6 +208,7 @@ static const char *const subdirs[] = {
 	"pg_snapshots",
 	"pg_subtrans",
 	"pg_twophase",
+	"pg_fdwxact",
 	"pg_multixact",
 	"pg_multixact/members",
 	"pg_multixact/offsets",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index e73639df74..3041c39bc0 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -302,6 +302,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 233441837f..b040202043 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -710,6 +710,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -914,6 +915,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/fdwxactdesc.c b/src/bin/pg_waldump/fdwxactdesc.c
new file mode 120000
index 0000000000..ce8c21880c
--- /dev/null
+++ b/src/bin/pg_waldump/fdwxactdesc.c
@@ -0,0 +1 @@
+../../../src/backend/access/rmgrdesc/fdwxactdesc.c
\ No newline at end of file
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca4b1..b616cea347 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
new file mode 100644
index 0000000000..5df32b2703
--- /dev/null
+++ b/src/include/access/fdwxact.h
@@ -0,0 +1,162 @@
+/*
+ * fdwxact.h
+ *
+ * PostgreSQL global transaction manager
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact.h
+ */
+#ifndef FDWXACT_H
+#define FDWXACT_H
+
+#include "access/fdwxact_xlog.h"
+#include "access/xlogreader.h"
+#include "foreign/foreign.h"
+#include "lib/stringinfo.h"
+#include "miscadmin.h"
+#include "nodes/pg_list.h"
+#include "nodes/execnodes.h"
+#include "storage/backendid.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "utils/guc.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* fdwXactState */
+#define	FDWXACT_NOT_WAITING		0
+#define	FDWXACT_WAITING			1
+#define	FDWXACT_WAIT_COMPLETE	2
+
+/* Flag passed to FDW transaction management APIs */
+#define FDWXACT_FLAG_ONEPHASE		0x01	/* transaction can commit/rollback
+											 * without preparation */
+
+/* Enum for foreign_twophase_commit parameter */
+typedef enum
+{
+	FOREIGN_TWOPHASE_COMMIT_DISABLED,	/* disable foreign twophase commit */
+	FOREIGN_TWOPHASE_COMMIT_REQUIRED	/* all foreign servers have to support
+										 * twophase commit */
+}			ForeignTwophaseCommitLevel;
+
+/* Enum to track the status of foreign transaction */
+typedef enum
+{
+	FDWXACT_STATUS_INVALID = 0,
+	FDWXACT_STATUS_PREPARING,	/* foreign transaction is being prepared */
+	FDWXACT_STATUS_PREPARED,	/* foreign transaction is prepared */
+	FDWXACT_STATUS_COMMITTING,	/* foreign prepared transaction is being
+								 * committed */
+	FDWXACT_STATUS_ABORTING		/* foreign prepared transaction is being
+								 * aborted */
+} FdwXactStatus;
+
+typedef struct FdwXactData *FdwXact;
+
+/*
+ * Shared memory state of a single foreign transaction.
+ */
+typedef struct FdwXactData
+{
+	FdwXact		fdwxact_free_next;	/* Next free FdwXact entry */
+
+	TransactionId local_xid;	/* XID of local transaction */
+	Oid			dbid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			umid;
+
+	FdwXactStatus status;
+	bool		indoubt;		/* Is an in-doubt transaction? */
+	bool		inprocessing;	/* resolver is processing? */
+	slock_t		mutex;			/* protect above three fields */
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXact. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXact. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;	/* XLOG offset of inserting this entry
+									 * start */
+	XLogRecPtr	insert_end_lsn; /* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to
+								 * file? */
+	BackendId	held_by;		/* backend who are holding */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* prepared transaction
+												 * identifier */
+}			FdwXactData;
+
+/*
+ * Shared memory layout for maintaining foreign prepared transaction entries.
+ * Adding or removing FdwXact entry needs to hold FdwXactLock in exclusive mode,
+ * and iterating fdwXacts needs that in shared mode.
+ */
+typedef struct
+{
+	/* Head of linked list of free FdwXactData structs */
+	FdwXact		free_fdwxacts;
+
+	/* Number of valid foreign transaction entries */
+	int			num_fdwxacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXact		fdwxacts[FLEXIBLE_ARRAY_MEMBER];	/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
+/* State data for foreign transaction resolution, passed to FDW callbacks */
+typedef struct FdwXactRslvState
+{
+	TransactionId xid;
+
+	/* Foreign transaction information */
+	char	   *fdwxact_id;
+	ForeignServer *server;
+	UserMapping *usermapping;
+
+	int			flags;			/* OR of FDWXACT_FLAG_xx flags */
+} FdwXactRslvState;
+
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+extern int	foreign_twophase_commit;
+
+/* Function declarations */
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
+extern void RegisterFdwXactByRelId(Oid relid, bool modified);
+extern void RegisterFdwXactByServerId(Oid serverid, bool modified);
+extern void FdwXactReleaseWaiter(PGPROC *waiter);
+extern void FdwXactWaitForResolution(TransactionId wait_xid);
+extern void FdwXactResolveFdwXacts(int *fdwxact_idxs, int nfdwxacts, PGPROC *waiter);
+extern PGPROC *FdwXactGetWaiter(TimestampTz now, TimestampTz *nextResolutionTs_p,
+								TransactionId *waitXid_p);
+extern bool FdwXactWaiterExists(Oid dbid);
+extern bool PrepareFdwXactParticipants(TransactionId xid);
+extern void SetFdwXactParticipants(TransactionId xid, bool commit);
+extern void PreCommit_FdwXact(void);
+extern void AtEOXact_FdwXact(bool is_commit);
+extern void AtPrepare_FdwXact(void);
+extern void PostPrepare_FdwXact(void);
+extern void FdwXactCleanupAtProcExit(void);
+extern void restoreFdwXactData(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
+extern void RecoverFdwXacts(void);
+extern bool FdwXactExists(Oid dboid, Oid serverid, Oid userid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+
+#endif							/* FDWXACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000000..688b43b8d0
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,28 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void FdwXactLauncherRequestToLaunch(void);
+extern void FdwXactLaunchOrWakeupResolver(void);
+extern Size FdwXactRslvShmemSize(void);
+extern void FdwXactRslvShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+
+
+#endif							/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000000..779848113c
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,23 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int	foreign_xact_resolver_timeout;
+
+#endif							/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000000..b4cec76eae
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,54 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDWXACT_INSERT	0x00
+#define XLOG_FDWXACT_REMOVE	0x10
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDWXACT_ID_MAX_LEN 200
+
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId local_xid;
+	Oid			dbid;			/* database oid where to find foreign server
+								 * and user mapping */
+	Oid			serverid;		/* foreign server where transaction takes
+								 * place */
+	Oid			userid;			/* user who initiated the foreign transaction */
+	Oid			umid;
+	char		fdwxact_id[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */
+} FdwXactOnDiskData;
+
+typedef struct xl_fdwxact_remove
+{
+	TransactionId xid;
+	Oid			serverid;
+	Oid			userid;
+	Oid			dbid;
+	bool		force;
+} xl_fdwxact_remove;
+
+extern void fdwxact_redo(XLogReaderState *record);
+extern void fdwxact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdwxact_identify(uint8 info);
+
+#endif							/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000000..c935471936
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,63 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2020, PostgreSQL Global Development Group
+ *
+ * src/include/access/resolver_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLaunchLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t		pid;			/* this resolver's PID, or 0 if not active */
+	Oid			dbid;			/* database oid */
+
+	/* Indicates if this slot is used of free */
+	bool		in_use;
+
+	/* Protect shared variables shown above */
+	slock_t		mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	   *latch;
+} FdwXactResolver;
+
+/* There is one FdwXactRslvCtlData struct for the whole database cluster */
+typedef struct FdwXactRslvCtlData
+{
+	/* Foreign transaction resolution queue. Protected by FdwXactLock */
+	SHM_QUEUE	fdwxact_queue;
+
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch	   *launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactRslvCtlData;
+#define SizeOfFdwXactRslvCtlData \
+	(offsetof(FdwXactRslvCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+extern FdwXactResolver *MyFdwXactResolver;
+extern FdwXactRslvCtlData *FdwXactRslvCtl;
+
+#endif							/* RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index 6c15df7e70..986bc73566 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Fdw Transaction", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 2ca71c3445..bd027a2861 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index a04fc70326..6f1f336e31 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -108,6 +108,13 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
 
+/*
+ * XACT_FLAGS_FDWNONPREPARE - set when we wrote data on foreign table of which
+ * server isn't capable of two-phase commit
+ * relation.
+ */
+#define XACT_FLAGS_FDWNOPREPARE					(1U << 3)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index c8869d5226..da0d442f1b 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -232,6 +232,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index de5670e538..9884f5f8e7 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index 61f2c2f5b4..df5189dd2d 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -5981,6 +5981,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '9706', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '1000', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{xid,oid,oid,text,bool,text}',
+  proargmodes => '{o,o,o,o,o,o}',
+  proargnames => '{xid,serverid,userid,status,in_doubt,identifier}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '9707', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '9708', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid oid',
+  proargnames => '{xid,serverid,userid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -6099,6 +6117,10 @@
 { oid => '2851', descr => 'wal filename, given a wal location',
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
+{ oid => '9709',
+  descr => 'stop a foreign transaction resolver process running on the given database',
+  proname => 'pg_stop_foreing_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'oid', prosrc => 'pg_stop_foreign_xact_resolver'},
 
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index 95556dfb15..8d046cc4e4 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -12,6 +12,7 @@
 #ifndef FDWAPI_H
 #define FDWAPI_H
 
+#include "access/fdwxact.h"
 #include "access/parallel.h"
 #include "nodes/execnodes.h"
 #include "nodes/pathnodes.h"
@@ -169,6 +170,11 @@ typedef bool (*IsForeignScanParallelSafe_function) (PlannerInfo *root,
 typedef List *(*ReparameterizeForeignPathByChild_function) (PlannerInfo *root,
 															List *fdw_private,
 															RelOptInfo *child_rel);
+typedef void (*PrepareForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*CommitForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef void (*RollbackForeignTransaction_function) (FdwXactRslvState *frstate);
+typedef char *(*GetPrepareId_function) (TransactionId xid, Oid serverid,
+										Oid userid, int *prep_id_len);
 
 /*
  * FdwRoutine is the struct returned by a foreign-data wrapper's handler
@@ -236,6 +242,12 @@ typedef struct FdwRoutine
 	/* Support functions for IMPORT FOREIGN SCHEMA */
 	ImportForeignSchema_function ImportForeignSchema;
 
+	/* Support functions for transaction management */
+	PrepareForeignTransaction_function PrepareForeignTransaction;
+	CommitForeignTransaction_function CommitForeignTransaction;
+	RollbackForeignTransaction_function RollbackForeignTransaction;
+	GetPrepareId_function GetPrepareId;
+
 	/* Support functions for parallelism under Gather node */
 	IsForeignScanParallelSafe_function IsForeignScanParallelSafe;
 	EstimateDSMForeignScan_function EstimateDSMForeignScan;
diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h
index 5e0cf533fb..5596ee591c 100644
--- a/src/include/foreign/foreign.h
+++ b/src/include/foreign/foreign.h
@@ -69,6 +69,7 @@ extern ForeignServer *GetForeignServerExtended(Oid serverid,
 											   bits16 flags);
 extern ForeignServer *GetForeignServerByName(const char *name, bool missing_ok);
 extern UserMapping *GetUserMapping(Oid userid, Oid serverid);
+extern UserMapping *GetUserMappingByOid(Oid umid);
 extern ForeignDataWrapper *GetForeignDataWrapper(Oid fdwid);
 extern ForeignDataWrapper *GetForeignDataWrapperExtended(Oid fdwid,
 														 bits16 flags);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index c55dc1481c..2186c1c5d0 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -806,6 +806,8 @@ typedef enum
 	WAIT_EVENT_BGWRITER_HIBERNATE,
 	WAIT_EVENT_BGWRITER_MAIN,
 	WAIT_EVENT_CHECKPOINTER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
 	WAIT_EVENT_LOGICAL_APPLY_MAIN,
 	WAIT_EVENT_LOGICAL_LAUNCHER_MAIN,
 	WAIT_EVENT_PGSTAT_MAIN,
@@ -853,6 +855,7 @@ typedef enum
 	WAIT_EVENT_CHECKPOINT_DONE,
 	WAIT_EVENT_CHECKPOINT_START,
 	WAIT_EVENT_EXECUTE_GATHER,
+	WAIT_EVENT_FDWXACT_RESOLUTION,
 	WAIT_EVENT_HASH_BATCH_ALLOCATE,
 	WAIT_EVENT_HASH_BATCH_ELECT,
 	WAIT_EVENT_HASH_BATCH_LOAD,
@@ -969,6 +972,9 @@ typedef enum
 	WAIT_EVENT_TWOPHASE_FILE_READ,
 	WAIT_EVENT_TWOPHASE_FILE_SYNC,
 	WAIT_EVENT_TWOPHASE_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
 	WAIT_EVENT_WALSENDER_TIMELINE_HISTORY_READ,
 	WAIT_EVENT_WAL_BOOTSTRAP_SYNC,
 	WAIT_EVENT_WAL_BOOTSTRAP_WRITE,
diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h
index 1ee9000b2b..4150d8a3e4 100644
--- a/src/include/storage/proc.h
+++ b/src/include/storage/proc.h
@@ -16,6 +16,7 @@
 
 #include "access/clog.h"
 #include "access/xlogdefs.h"
+#include "datatype/timestamp.h"
 #include "lib/ilist.h"
 #include "storage/latch.h"
 #include "storage/lock.h"
@@ -154,6 +155,16 @@ struct PGPROC
 	int			syncRepState;	/* wait state for sync rep */
 	SHM_QUEUE	syncRepLinks;	/* list link if process is in syncrep queue */
 
+	/*
+	 * Info to allow us to wait for foreign transaction to be resolved, if
+	 * needed.
+	 */
+	TransactionId	fdwXactWaitXid;	/* waiting for foreign transaction involved with
+									 * this transaction id to be resolved */
+	int			fdwXactState;	/* wait state for foreign transaction resolution */
+	SHM_QUEUE	fdwXactLinks;	/* list link if process is in queue */
+	TimestampTz fdwXactNextResolutionTs;
+
 	/*
 	 * All PROCLOCK objects for locks held or awaited by this backend are
 	 * linked into one of these lists, according to the partition number of
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index a5c7d0c064..0f73b64937 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -36,6 +36,8 @@
 
 #define		PROCARRAY_SLOTS_XMIN			0x20	/* replication slot xmin,
 													 * catalog_xmin */
+#define		PROCARRAY_FDWXACT_XMIN			0x40	/* unresolved distributed
+													   transaciton xmin */
 /*
  * Only flags in PROCARRAY_PROC_FLAGS_MASK are considered when matching
  * PGXACT->vacuumFlags. Other flags are used for different purposes and
@@ -125,4 +127,7 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
 
+
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
+extern TransactionId ProcArrayGetFdwXactUnresolvedXmin(void);
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 454c2df487..f977ca43d4 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -96,6 +96,8 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FOREIGN_TRANSACTION,
+	FOREIGN_TRANSACTION_RESOLVER,
 	COMPAT_OPTIONS,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index b813e32215..628eaf531e 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1342,6 +1342,13 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.xid,
+    f.serverid,
+    f.userid,
+    f.status,
+    f.in_doubt,
+    f.identifier
+   FROM pg_foreign_xacts() f(xid, serverid, userid, status, in_doubt, identifier);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
-- 
2.23.0

From d2156de537f7b2cb1d01a6a1f5a58c73017b00d0 Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Tue, 19 May 2020 15:06:38 +0900
Subject: [PATCH v22 3/6] Documentation update.

---
 doc/src/sgml/catalogs.sgml                | 136 +++++++++++++
 doc/src/sgml/config.sgml                  | 144 +++++++++++++
 doc/src/sgml/distributed-transaction.sgml | 143 +++++++++++++
 doc/src/sgml/fdwhandler.sgml              | 236 ++++++++++++++++++++++
 doc/src/sgml/filelist.sgml                |   1 +
 doc/src/sgml/func.sgml                    |  89 ++++++++
 doc/src/sgml/monitoring.sgml              |  42 ++++
 doc/src/sgml/postgres.sgml                |   1 +
 doc/src/sgml/storage.sgml                 |   6 +
 9 files changed, 798 insertions(+)
 create mode 100644 doc/src/sgml/distributed-transaction.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 700271fd40..2af2eda512 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -9223,6 +9223,11 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
       <entry>summary of configuration file contents</entry>
      </row>
 
+     <row>
+      <entry><link linkend="view-pg-foreign-xacts"><structname>pg_foreign_xacts</structname></link></entry>
+      <entry>foreign transactions</entry>
+     </row>
+
      <row>
       <entry><link linkend="view-pg-group"><structname>pg_group</structname></link></entry>
       <entry>groups of database users</entry>
@@ -10934,6 +10939,137 @@ SELECT * FROM pg_locks pl LEFT JOIN pg_prepared_xacts ppx
 
  </sect1>
 
+ <sect1 id="view-pg-foreign-xacts">
+  <title><structname>pg_foreign_xacts</structname></title>
+
+  <indexterm zone="view-pg-foreign-xacts">
+   <primary>pg_foreign_xacts</primary>
+  </indexterm>
+
+  <para>
+   The view <structname>pg_foreign_xacts</structname> displays
+   information about foreign transactions that are opened on
+   foreign servers for atomic distributed transaction commit (see
+   <xref linkend="atomic-commit"/> for details).
+  </para>
+
+  <para>
+   <structname>pg_foreign_xacts</structname> contains one row per foreign
+   transaction.  An entry is removed when the foreign transaction is
+   committed or rolled back.
+  </para>
+
+  <table>
+   <title><structname>pg_foreign_xacts</structname> Columns</title>
+
+   <tgroup cols="4">
+    <thead>
+     <row>
+      <entry>Name</entry>
+      <entry>Type</entry>
+      <entry>References</entry>
+      <entry>Description</entry>
+     </row>
+    </thead>
+    <tbody>
+     <row>
+      <entry><structfield>dbid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
+      <entry>
+       OID of the database which the foreign transaction resides in
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>xid</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>
+       Numeric transaction identifier with which this foreign transaction
+       associates
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>serverid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="catalog-pg-foreign-server"><structname>pg_foreign_server</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the foreign server on which the foreign transaction is prepared
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>userid</structfield></entry>
+      <entry><type>oid</type></entry>
+      <entry><literal><link linkend="view-pg-user"><structname>pg_user</structname></link>.oid</literal></entry>
+      <entry>
+       The OID of the user that prepared this foreign transaction.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>status</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       Status of foreign transaction. Possible values are:
+       <itemizedlist>
+        <listitem>
+         <para>
+          <literal>preparing</literal> : This foreign transaction is being prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>prepared</literal> : This foreign transaction has been prepared.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>committing</literal> : This foreign transcation is being committed.
+         </para>
+        </listitem>
+        <listitem>
+         <para>
+          <literal>aborting</literal> : This foreign transaction is being aborted.
+         </para>
+        </listitem>
+       </itemizedlist>
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>in_doubt</structfield></entry>
+      <entry><type>boolean</type></entry>
+      <entry></entry>
+      <entry>
+       If <literal>true</literal> this foreign transaction is in-doubt status.
+       A foreign transaction becomes in-doubt status when user canceled the
+       query during transaction commit or the server crashed during transaction
+       commit.
+      </entry>
+     </row>
+     <row>
+      <entry><structfield>identifier</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>
+       The identifier of the prepared foreign transaction.
+      </entry>
+     </row>
+    </tbody>
+   </tgroup>
+  </table>
+
+  <para>
+   When the <structname>pg_foreign_xacts</structname> view is accessed, the
+   internal transaction manager data structures are momentarily locked, and
+   a copy is made for the view to display.  This ensures that the
+   view produces a consistent set of results, while not blocking
+   normal operations longer than necessary.  Nonetheless
+   there could be some impact on database performance if this view is
+   frequently accessed.
+  </para>
+
+ </sect1>
+
  <sect1 id="view-pg-publication-tables">
   <title><structname>pg_publication_tables</structname></title>
 
diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml
index 31b4660160..918fac967c 100644
--- a/doc/src/sgml/config.sgml
+++ b/doc/src/sgml/config.sgml
@@ -9085,6 +9085,150 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir'
      </variablelist>
    </sect1>
 
+   <sect1 id="runtime-config-distributed-transaction">
+    <title>Distributed Transaction Management</title>
+
+    <sect2 id="runtime-config-distributed-transaction-settings">
+     <title>Setting</title>
+     <variablelist>
+
+      <varlistentry id="guc-foreign-twophase-commit" xreflabel="foreign_twophse_commit">
+       <term><varname>foreign_twophase_commit</varname> (<type>enum</type>)
+        <indexterm>
+         <primary><varname>foreign_twophase_commit</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies whether distributed transaction commits ensures that all
+         involved changes on foreign servers are committed or not. Valid
+         values are <literal>required</literal> and <literal>disabled</literal>.
+         The default setting is <literal>disabled</literal>. Setting to
+         <literal>disabled</literal> don't use two-phase commit protocol to
+         commit or rollback distributed transactions. When set to
+         <literal>required</literal> distributed transactions strictly requires
+         that all written servers can use two-phase commit protocol.  That is,
+         the distributed transaction cannot commit if even one server does not
+         support the prepare callback routine
+         (described in <xref linkend="fdw-callbacks-transaction-managements"/>).
+         In <literal>required</literal> case, distributed transaction commit will
+         wait for all involving foreign transaction to be committed before the
+         command return a "success" indication to the client.
+        </para>
+
+        <para>
+         This parameter can be changed at any time; the behavior for any one
+         transaction is determined by the setting in effect when it commits.
+        </para>
+
+        <note>
+         <para>
+          When <literal>disabled</literal> there can be risk of database
+          consistency among all servers that involved in the distributed
+          transaction when some foreign server crashes during committing the
+          distributed transaction.
+         </para>
+        </note>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-max-prepared-foreign-transactions" xreflabel="max_prepared_foreign_transactions">
+       <term><varname>max_prepared_foreign_transactions</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>max_prepared_foreign_transactions</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Sets the maximum number of foreign transactions that can be prepared
+         simultaneously. A single local transaction can give rise to multiple
+         foreign transaction. If a user expects <literal>N</literal> local
+         transactions and each of those involves <literal>K</literal> foreign
+         servers, this value need to be set <literal>N * K</literal>, not
+         just <literal>N</literal>.  This parameter can only be set at server
+         start.
+        </para>
+        <para>
+         When running a standby server, you must set this parameter to the
+         same or higher value than on the master server. Otherwise, queries
+         will not be allowed in the standby server.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+    </sect2>
+
+    <sect2 id="runtime-config-foreign-transaction-resolver">
+     <title>Foreign Transaction Resolvers</title>
+
+     <para>
+      These settings control the behavior of a foreign transaction resolver.
+     </para>
+
+     <variablelist>
+      <varlistentry id="guc-max-foreign-transaction-resolvers" xreflabel="max_foreign_transaction_resolvers">
+       <term><varname>max_foreign_transaction_resolvers</varname> (<type>int</type>)
+        <indexterm>
+         <primary><varname>max_foreign_transaction_resolvers</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specifies maximum number of foreign transaction resolution workers. A foreign transaction
+         resolver is responsible for foreign transaction resolution on one database.
+        </para>
+        <para>
+         Foreign transaction resolution workers are taken from the pool defined by
+         <varname>max_worker_processes</varname>.
+        </para>
+        <para>
+         The default value is 0.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolution-rety-interval" xreflabel="foreign_transaction_resolution_retry_interval">
+       <term><varname>foreign_transaction_resolution_retry_interval</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolution_interval</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Specify how long the foreign transaction resolver should wait when the last resolution
+         fails before retrying to resolve foreign transaction. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 10 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+
+      <varlistentry id="guc-foreign-transaction-resolver-timeout" xreflabel="foreign_transaction_resolver_timeout">
+       <term><varname>foreign_transaction_resolver_timeout</varname> (<type>integer</type>)
+        <indexterm>
+         <primary><varname>foreign_transaction_resolver_timeout</varname> configuration parameter</primary>
+        </indexterm>
+       </term>
+       <listitem>
+        <para>
+         Terminate foreign transaction resolver processes that don't have any foreign
+         transactions to resolve longer than the specified number of milliseconds.
+         A value of zero disables the timeout mechanism, meaning it connects to one
+         database until stopping manually. This parameter can only be set in the
+         <filename>postgresql.conf</filename> file or on the server command line.
+        </para>
+        <para>
+         The default value is 60 seconds.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </sect2>
+   </sect1>
+
    <sect1 id="runtime-config-compatible">
     <title>Version and Platform Compatibility</title>
 
diff --git a/doc/src/sgml/distributed-transaction.sgml b/doc/src/sgml/distributed-transaction.sgml
new file mode 100644
index 0000000000..139dd7f918
--- /dev/null
+++ b/doc/src/sgml/distributed-transaction.sgml
@@ -0,0 +1,143 @@
+<!-- doc/src/sgml/distributed-transaction.sgml -->
+
+<chapter id="distributed-transaction">
+ <title>Distributed Transaction</title>
+
+ <para>
+  A distributed transaction is a transaction in which two or more network hosts
+  are involved. <productname>PostgreSQL</productname>'s global Transaction
+  manager supports distributed transactions that access foreign servers using
+  Foreign Data Wrappers. The global transaction manager is responsible for
+  managing transactions on foreign servers.
+ </para>
+
+ <sect1 id="atomic-commit">
+  <title>Atomic Commit</title>
+
+  <para>
+   Formerly, transactions on foreign server were simply committed or rolled
+   back one by one. Therefore, when one foreign server had a problem during
+   commit, it was possible that transactions on only part of foreign servers
+   are committed while other transactions are rolled back. This used to leave
+   database data in an inconsistent state in term of federated database.
+   Atomic commit of distributed transaction is an operation that applies a set
+   of changes as a single operation globally. This guarantees all-or-nothing
+   results for the changes on all remote hosts involved in.
+   <productname>PostgreSQL</productname> provides a way to perform read-write
+   transactions with foreign resources using foreign data wrappers.
+   Using the <productname>PostgreSQL</productname>'s atomic commit ensures that
+   all changes on foreign servers end in either commit or rollback using the
+   transaction callback routines
+   (see <xref linkend="fdw-callbacks-transaction-managements"/>).
+  </para>
+
+  <sect2>
+   <title>Atomic Commit Using Two-phase Commit Protocol</title>
+
+   <para>
+    To achieve commit among all foreign servers automatically,
+    <productname>PostgreSQL</productname> employs two-phase commit protocol,
+    which is a type of atomic commitment protocol (ACP).  Using two-phase
+    commit protocol, the commit sequence of distributed transaction performs
+    with the following steps:
+    <orderedlist>
+     <listitem>
+      <para>
+       Prepare all transactions on foreign servers.
+       <productname>PostgreSQL</productname> distributed transaction manager
+       prepares all transaction on the foreign servers if two-phase commit is
+       required. Two-phase commit is required when the transaction modifies
+       data on two or more servers including the local server itself and
+       <xref linkend="guc-foreign-twophase-commit"/> is
+       <literal>required</literal>. If all preparations on foreign servers got
+       successful go to the next step. Any failure happens in this step,
+       the server changes to rollback, then rollback all transactions on both
+       local and foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Commit locally. The server commits transaction locally.  Any failure happens
+       in this step the server changes to rollback, then rollback all transactions
+       on both local and foreign servers.
+      </para>
+     </listitem>
+     <listitem>
+      <para>
+       Resolve all prepared transaction on foreign servers. Pprepared transactions
+       are committed or rolled back according to the result of the local transaction.
+       This step is normally performed by a foreign transaction resolver process.
+      </para>
+     </listitem>
+    </orderedlist>
+   </para>
+
+   <para>
+    Each commit of a distributed transaction will wait until confirmation is
+    received that all prepared transactions are committed or rolled back.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-in-doubt-transaction">
+   <title>In-Doubt Transactions</title>
+
+   <para>
+    The atomic commit mechanism ensures that all foreign servers either commit
+    or rollback using two-phase commit protocol. However, foreign transactions
+    become <firstterm>in-doubt</firstterm> in three cases: where the foreign
+    server crashed or lost the connectibility to it during preparing foreign
+    transaction, where the local node crashed during either preparing or
+    resolving foreign transaction and where user canceled the query. You can
+    check in-doubt transaction in <xref linkend="view-pg-foreign-xacts"/>
+    view. These foreign transactions are resolved by foreign transaction resolver
+    process or executing <function>pg_resolve_foriegn_xact</function> function
+    manually.
+   </para>
+  </sect2>
+
+  <sect2 id="atomic-commit-transaction-resolver">
+   <title>Foreign Transaction Resolver Processes</title>
+
+   <para>
+    Foreign transaction resolver processes are auxiliary processes that are
+    responsible for resolving both foreign transactions that are prepared by
+    online transactions and in-doubt transactions. They commit or rollback
+    prepared transactions on all foreign servers involved with the distributed
+    transaction if the local node received agreement messages from all
+    foreign servers during the first step of two-phase commit protocol.
+   </para>
+
+   <para>
+    One foreign transaction resolver is responsible for transaction resolutions
+    on which one database connecting. On failure during resolution, they retry to
+    resolve at an interval of <varname>foreign_transaction_resolution_interval</varname>
+    time.
+   </para>
+
+   <note>
+    <para>
+     During a foreign transaction resolver process connecting to the database,
+     database cannot be dropped without immediate shutdown. You can call
+     <function>pg_stop_foreign_xact_resovler</function> function to stop the
+     particular resolver process before dropping the database.
+    </para>
+   </note>
+  </sect2>
+
+  <sect2>
+   <title>Configuration Settings</title>
+
+   <para>
+    Atomic commit requires several configuration options to be set.
+    On the local node, <xref linkend="guc-max-prepared-foreign-transactions"/> and
+    <xref linkend="guc-max-foreign-transaction-resolvers"/> must be non-zero value.
+    Additionally the <varname>max_worker_processes</varname> may need to be adjusted to
+    accommodate for foreign transaction resolver workers, at least
+    (<varname>max_foreign_transaction_resolvers</varname> + <literal>1</literal>).
+    Note that other <productname>PostgreSQL</productname> feature such as parallel
+    queries, logical replication, etc., also take worker slots from
+    <varname>max_worker_processes</varname>.
+   </para>
+  </sect2>
+ </sect1>
+</chapter>
diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml
index 6587678af2..dd0358ef22 100644
--- a/doc/src/sgml/fdwhandler.sgml
+++ b/doc/src/sgml/fdwhandler.sgml
@@ -1415,6 +1415,127 @@ ReparameterizeForeignPathByChild(PlannerInfo *root, List *fdw_private,
     </para>
    </sect2>
 
+   <sect2 id="fdw-callbacks-transaction-managements">
+    <title>FDW Routines For Transaction Managements</title>
+
+    <para>
+     Transaction management callbacks are used for doing commit, rollback and
+     prepare the foreign transaction. If an FDW wishes that its foreign
+     transaction is managed by <productname>PostgreSQL</productname>'s global
+     transaction manager it must provide both
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>. In addition, if an FDW
+     wishes to support <firstterm>atomic commit</firstterm> (as described in
+     <xref linkend="fdw-transaction-managements"/>), it must provide
+     <function>PrepareForeignTransaction</function> as well and can provide
+     <function>GetPrepareId</function> callback optionally.
+    </para>
+
+    <para>
+<programlisting>
+void
+PrepareForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Prepare the transaction on the foreign server. This function is called at the
+    pre-commit phase of the local transactions if foreign twophase commit is
+    required. This function is used only for distribute transaction management
+    (see <xref linkend="distributed-transaction"/>).
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+    <para>
+<programlisting>
+bool
+CommitForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Commit the foreign transaction. This function is called either at
+    the pre-commit phase of the local transaction if the transaction
+    can be committed in one-phase or at the post-commit phase if
+    two-phase commit is required. If <literal>frstate-&gt;flag</literal> has
+    the flag <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction
+    can be committed in one-phase, this function must commit the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+bool
+RollbackForeignTransaction(FdwXactRslvState *frstate);
+</programlisting>
+    Rollback the foreign transaction. This function is called either at
+    the end of the local transaction after rolled back locally. The foreign
+    transactions are rolled back when user requested rollbacking or when
+    any error occurs during the transaction. This function must be tolerate to
+    being called recursively if any error occurs during rollback the foreign
+    transaction. So you would need to track recursion and prevent being called
+    infinitely. If <literal>frstate-&gt;flag</literal> has the flag
+    <literal>FDW_XACT_FLAG_ONEPHASE</literal> the transaction can be rolled
+    back in one-phase, otherwise this function must rollback the prepared
+    transaction identified by <literal>frstate-&gt;fdwxact_id</literal>.
+    </para>
+
+    <para>
+     The foreign transaction identified by <literal>frstate-&gt;fdwxact_id</literal>
+     might not exist on the foreign servers. This can happen when, for instance,
+     <productname>PostgreSQL</productname> server crashed during preparing or
+     committing the foreign tranasction. Therefore, this function needs to
+     tolerate the undefined object error
+     (<literal>ERRCODE_UNDEFINED_OBJECT</literal>) rather than raising an error.
+    </para>
+
+    <para>
+     Note that all cases except for calling <function>pg_resolve_fdwxact</function>
+     SQL function, this callback function is executed by foreign transaction
+     resolver processes.
+    </para>
+    <para>
+<programlisting>
+char *
+GetPrepareId(TransactionId xid, Oid serverid, Oid userid, int *prep_id_len);
+</programlisting>
+    Return null terminated string that represents prepared transaction identifier
+    with its length <varname>*prep_id_len</varname>.
+    This optional function is called during executor startup for once per the
+    foreign server. Note that the transaction identifier must be string literal,
+    less than <symbol>NAMEDATALEN</symbol> bytes long and should not be same
+    as any other concurrent prepared transaction id. If this callback routine
+    is not supported, <productname>PostgreSQL</productname>'s  distributed
+    transaction manager generates an unique identifier with in the form of
+    <literal>fx_&lt;random value up to 2<superscript>31</superscript>&gt;_&lt;server oid&gt;_&lt;user oid&gt;</literal>.
+    </para>
+
+    <para>
+     Note that this callback function is always executed by backend processes.
+    </para>
+
+    <note>
+     <para>
+      Functions <function>PrepareForeignTransaction</function>,
+      <function>CommitForeignTransaction</function> and
+      <function>RolblackForeignTransaction</function> are called
+      at outside of a valid transaction state. So please note that
+      you cannot use functions that use the system catalog cache
+      such as Foreign Data Wrapper helper functions described in
+      <xref linkend="fdw-helpers"/>.
+     </para>
+    </note>
+   </sect2>
    </sect1>
 
    <sect1 id="fdw-helpers">
@@ -1894,4 +2015,119 @@ GetForeignServerByName(const char *name, bool missing_ok);
 
   </sect1>
 
+  <sect1 id="fdw-transaction-managements">
+   <title>Transaction managements for Foreign Data Wrappers</title>
+   <para>
+    If a FDW's server supports transaction, it is usually worthwhile for the
+    FDW to manage transaction opened on the foreign server. The FDW callback
+    function <literal>CommitForeignTransaction</literal>,
+    <literal>RollbackForeignTransaction</literal> and
+    <literal>PrepareForeignTransaction</literal> are used to manage Transaction
+    management and must fit into the working of the
+    <productname>PostgreSQL</productname> transaction processing.
+   </para>
+
+   <para>
+    The information in <literal>FdwXactRslvState</literal> can be used to get
+    information of foreign server being processed such as server name, OID of
+    server, user and user mapping. The <literal>flags</literal> has contains flag
+    bit describing the foreign transaction state for transaction management.
+   </para>
+
+   <sect2 id="fdw-transaction-commit-rollback">
+    <title>Commit And Rollback Single Foreign Transaction</title>
+    <para>
+     The FDW callback function <literal>CommitForeignTransaction</literal>
+     and <literal>RollbackForeignTransaction</literal> can be used to commit
+     and rollback the foreign transaction. During transaction commit, the core
+     transaction manager calls <literal>CommitForeignTransaction</literal> function
+     in the pre-commit phase and calls
+     <literal>RollbackForeignTransaction</literal> function in the post-rollback
+     phase.
+    </para>
+   </sect2>
+
+   <sect2 id="fdw-transaction-distributed-transaction-commit">
+    <title>Atomic Commit And Rollback Distributed Transaction</title>
+    <para>
+     In addition to simply commit and rollback foreign transactions described at
+     <xref linkend="fdw-transaction-commit-rollback"/>,
+     <productname>PostgreSQL</productname> global transaction manager enables
+     distributed transactions to atomically commit and rollback among all foreign
+     servers, which is as known as atomic commit in literature. To achieve atomic
+     commit, <productname>PostgreSQL</productname> employs two-phase commit
+     protocol, which is a type of atomic commitment protocol. Every FDWs that wish
+     to support two-phase commit protocol are required to have the FDW callback
+     function <function>PrepareForeignTransaction</function> and optionally
+     <function>GetPrepareId</function>, in addition to
+     <function>CommitForeignTransaction</function> and
+     <function>RollbackForeignTransaction</function>
+     (see <xref linkend="fdw-callbacks-transaction-managements"/> for details).
+    </para>
+
+    <para>
+     An example of distributed transaction is as follows
+<programlisting>
+BEGIN;
+UPDATE ft1 SET col = 'a';
+UPDATE ft2 SET col = 'b';
+COMMIT;
+</programlisting>
+    ft1 and ft2 are foreign tables on different foreign servers may be using different
+    Foreign Data Wrappers.
+    </para>
+
+    <para>
+     When the core executor access the foreign servers, foreign servers whose FDW
+     supports transaction management callback routines is registered as a participant.
+     During registration, <function>GetPrepareId</function> is called if provided to
+     generate an unique transaction identifer.
+    </para>
+
+    <para>
+     During pre-commit phase of local transaction, the foreign transaction manager
+     persists the foreign transaction information to the disk and WAL, and then
+     prepare all foreign transaction by calling
+     <function>PrepareForeignTransaction</function> if two-phase commit protocol
+     is required. Two-phase commit is required when the transaction modified data
+     on more than one servers including the local server itself and user requests
+     foreign twophase commit (see <xref linkend="guc-foreign-twophase-commit"/>).
+    </para>
+
+    <para>
+     <productname>PostgreSQL</productname> can commit locally and go to the next
+     step if and only if all foreign transactions are prepared successfully.
+     If any failure happens or user requests to cancel during preparation,
+     the distributed transaction manager changes over rollback and calls
+     <function>RollbackForeignTransaction</function>.
+    </para>
+
+    <para>
+     Note that when <literal>(frstate-&gt;flags &amp; FDWXACT_FLAG_ONEPHASE)</literal>
+     is true, both <literal>CommitForeignTransaction</literal> function and
+     <literal>RollbackForeignTransaction</literal> function should commit and
+     rollback directly, rather than processing prepared transactions. This can
+     happen when two-phase commit is not required or foreign server is not
+     modified with in the transaction.
+    </para>
+
+    <para>
+     Once all foreign transaction is prepared, the core transaction manager commits
+     locally. After that the transaction commit waits for all prepared foreign
+     transaction to be committed before completetion. After all prepared foreign
+     transactions are resolved the transaction commit completes.
+    </para>
+
+    <para>
+     One foreign transaction resolver process is responsible for foreign
+     transaction resolution on a database. Foreign transaction resolver process
+     calls either <function>CommitForeignTransaction</function> or
+     <function>RollbackForeignTransaction</function> to resolve foreign
+     transaction identified by <literal>frstate-&gt;fdwxact_id</literal>. If failed
+     to resolve, resolver process will exit with an error message. The foreign
+     transaction launcher will launch the resolver process again at
+     <xref linkend="guc-foreign-transaction-resolution-rety-interval"/> interval.
+    </para>
+   </sect2>
+  </sect1>
  </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 68179f71cd..1ab8e80fdc 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -48,6 +48,7 @@
 <!ENTITY wal           SYSTEM "wal.sgml">
 <!ENTITY logical-replication    SYSTEM "logical-replication.sgml">
 <!ENTITY jit    SYSTEM "jit.sgml">
+<!ENTITY distributed-transaction    SYSTEM "distributed-transaction.sgml">
 
 <!-- programmer's guide -->
 <!ENTITY bgworker   SYSTEM "bgworker.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index 7c06afd3ea..e281bd33d8 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -26126,6 +26126,95 @@ SELECT convert_from(pg_read_binary_file('file_in_utf8.txt'), 'UTF8');
 
   </sect2>
 
+  <sect2 id="functions-foreign-transaction">
+   <title>Foreign Transaction Management Functions</title>
+
+   <indexterm>
+    <primary>pg_resolve_foreign_xact</primary>
+   </indexterm>
+   <indexterm>
+    <primary>pg_remove_foreign_xact</primary>
+   </indexterm>
+
+   <para>
+    <xref linkend="functions-fdw-transaction-control-table"/> shows the functions
+    available for foreign transaction management.
+    These functions cannot be executed during recovery. Use of these function
+    is restricted to superusers.
+   </para>
+
+   <table id="functions-fdw-transaction-control-table">
+    <title>Foreign Transaction Management Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_resolve_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>userid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Resolve a foreign transaction. This function searches for foreign
+        transaction matching the arguments and resolves it. Once the foreign
+        transaction is resolved successfully, this function removes the
+        corresponding entry from <xref linkend="view-pg-foreign-xacts"/>.
+        This function won't resolve a foreign transaction which is being
+        processed.
+       </entry>
+      </row>
+      <row>
+       <entry>
+        <literal><function>pg_remove_foreign_xact(<parameter>transaction</parameter> <type>xid</type>, <parameter>serverid</parameter> <type>oid</type>, <parameter>userid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>void</type></entry>
+       <entry>
+        This function works the same as <function>pg_resolve_foreign_xact</function>
+        except that this removes the foreign transcation entry without resolution.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+   The function shown in <xref linkend="functions-fdwxact-resolver-control-table"/>
+   control the foreign transaction resolvers.
+   </para>
+
+   <table id="functions-fdwxact-resolver-control-table">
+    <title>Foreign Transaction Resolver Control Functions</title>
+    <tgroup cols="3">
+     <thead>
+      <row>
+       <entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry>
+      </row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <literal><function>pg_stop_fdwxact_resolver(<parameter>dbid</parameter> <type>oid</type>)</function></literal>
+       </entry>
+       <entry><type>bool</type></entry>
+       <entry>
+        Stop the foreign transaction resolver running on the given database.
+        This function is useful for stopping a resolver process on the database
+        that you want to drop.
+       </entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
+   <para>
+    <function>pg_stop_fdwxact_resolver</function> is useful to be used before
+    dropping the database to that the foreign transaction resolver is connecting.
+   </para>
+
+  </sect2>
   </sect1>
 
   <sect1 id="functions-trigger">
diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml
index 89662cc0a3..82c94d5e5d 100644
--- a/doc/src/sgml/monitoring.sgml
+++ b/doc/src/sgml/monitoring.sgml
@@ -1052,6 +1052,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       <entry><literal>CheckpointerMain</literal></entry>
       <entry>Waiting in main loop of checkpointer process.</entry>
      </row>
+     <row>
+      <entry><literal>FdwXactLauncherMain</literal></entry>
+      <entry>Waiting in main loop of foreign transaction resolution launcher process.</entry>
+     </row>
+     <row>
+      <entry><literal>FdwXactResolverMain</literal></entry>
+      <entry>Waiting in main loop of foreign transaction resolution worker process.</entry>
+     </row>
+     <row>
+      <entry><literal>LogicalLauncherMain</literal></entry>
+      <entry>Waiting in main loop of logical launcher process.</entry>
+     </row>
      <row>
       <entry><literal>LogicalApplyMain</literal></entry>
       <entry>Waiting in main loop of logical replication apply process.</entry>
@@ -1273,6 +1285,18 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       <entry><literal>DataFileWrite</literal></entry>
       <entry>Waiting for a write to a relation data file.</entry>
      </row>
+     <row>
+      <entry><literal>FdwXactFileRead</literal></entry>
+      <entry>Waiting for a read of a foreign transaction state file.</entry>
+     </row>
+     <row>
+      <entry><literal>FdwXactFileSync</literal></entry>
+      <entry>Waiting for a foreign transaction state file to reach stable storage.</entry>
+     </row>
+     <row>
+      <entry><literal>FdwXactFileWrite</literal></entry>
+      <entry>Waiting for a write of a foreign transaction state file.</entry>
+     </row>
      <row>
       <entry><literal>LockFileAddToDataDirRead</literal></entry>
       <entry>Waiting for a read while adding a line to the data directory lock
@@ -1550,6 +1574,11 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       <entry>Waiting for activity from a child process while
        executing a <literal>Gather</literal> plan node.</entry>
      </row>
+     <row>
+      <entry><literal>FdwXactResolution</literal></entry>
+      <entry>Waiting for all foreign transaction participants to be resolved during
+       atomic commit among foreign servers.</entry>
+     </row>
      <row>
       <entry><literal>HashBatchAllocate</literal></entry>
       <entry>Waiting for an elected Parallel Hash participant to allocate a hash
@@ -1861,6 +1890,19 @@ postgres   27093  0.0  0.0  30096  2752 ?        Ss   11:34   0:00 postgres: ser
       <entry>Waiting to read or update dynamic shared memory allocation
        information.</entry>
      </row>
+     <row>
+      <entry><literal>FdwXactLock</literal></entry>
+      <entry>Waiting to read or update the state of foreign transactions.</entry>
+     </row>
+     <row>
+      <entry><literal>FdwXactResolutionLock</literal></entry>
+      <entry>Waiting to read or update information of foreign trasnaction
+       resolution.</entry>
+     </row>
+     <row>
+      <entry><literal>LogicalRepWorkerLock</literal></entry>
+      <entry>Waiting for action on logical replication worker to finish.</entry>
+     </row>
      <row>
       <entry><literal>LockFastPath</literal></entry>
       <entry>Waiting to read or update a process' fast-path lock
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index c41ce9499b..5ef1f4a329 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -170,6 +170,7 @@ break is not needed in a wider output rendering.
   &wal;
   &logical-replication;
   &jit;
+  &distributed-transaction;
   &regress;
 
  </part>
diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml
index ea08d0b614..58f1e4fd15 100644
--- a/doc/src/sgml/storage.sgml
+++ b/doc/src/sgml/storage.sgml
@@ -83,6 +83,12 @@ Item
   subsystem</entry>
 </row>
 
+<row>
+ <entry><filename>pg_fdwxact</filename></entry>
+ <entry>Subdirectory containing files used by the distributed transaction
+  manager subsystem</entry>
+</row>
+
 <row>
  <entry><filename>pg_logical</filename></entry>
  <entry>Subdirectory containing status data for logical decoding</entry>
-- 
2.23.0

From 4c3b45ea7b027bf734bbf113d41de0a100dda49a Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <masahiko.sawada@2ndquadrant.com>
Date: Thu, 26 Mar 2020 14:12:17 +0500
Subject: [PATCH v22 1/6] Keep track of writing on non-temporary relation

---
 src/backend/executor/nodeModifyTable.c | 16 ++++++++++++++++
 src/include/access/xact.h              |  6 ++++++
 2 files changed, 22 insertions(+)

diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c
index 20a4c474cc..1ec07bad07 100644
--- a/src/backend/executor/nodeModifyTable.c
+++ b/src/backend/executor/nodeModifyTable.c
@@ -581,6 +581,10 @@ ExecInsert(ModifyTableState *mtstate,
 										   NULL,
 										   specToken);
 
+			/* Make note that we've wrote on non-temporary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			recheckIndexes = ExecInsertIndexTuples(slot, estate, true,
 												   &specConflict,
@@ -619,6 +623,10 @@ ExecInsert(ModifyTableState *mtstate,
 							   estate->es_output_cid,
 							   0, NULL);
 
+			/* Make note that we've wrote on non-temporary relation */
+			if (RelationNeedsWAL(resultRelationDesc))
+				MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 			/* insert index entries for tuple */
 			if (resultRelInfo->ri_NumIndices > 0)
 				recheckIndexes = ExecInsertIndexTuples(slot, estate, false, NULL,
@@ -970,6 +978,10 @@ ldelete:;
 	if (tupleDeleted)
 		*tupleDeleted = true;
 
+	/* Make note that we've wrote on non-temporary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/*
 	 * If this delete is the result of a partition key update that moved the
 	 * tuple to a new partition, put this row into the transition OLD TABLE,
@@ -1482,6 +1494,10 @@ lreplace:;
 	if (canSetTag)
 		(estate->es_processed)++;
 
+	/* Make note that we've wrote on non-temporary relation */
+	if (RelationNeedsWAL(resultRelationDesc))
+		MyXactFlags |= XACT_FLAGS_WROTENONTEMPREL;
+
 	/* AFTER ROW UPDATE Triggers */
 	ExecARUpdateTriggers(estate, resultRelInfo, tupleid, oldtuple, slot,
 						 recheckIndexes,
diff --git a/src/include/access/xact.h b/src/include/access/xact.h
index 7ee04babc2..a04fc70326 100644
--- a/src/include/access/xact.h
+++ b/src/include/access/xact.h
@@ -102,6 +102,12 @@ extern int	MyXactFlags;
  */
 #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK	(1U << 1)
 
+/*
+ * XACT_FLAGS_WROTENONTEMPREL - set when we wrote data on non-temporary
+ * relation.
+ */
+#define XACT_FLAGS_WROTENONTEMPREL				(1U << 2)
+
 /*
  *	start- and end-of-transaction callbacks for dynamically loaded modules
  */
-- 
2.23.0