From cc438a3fb50835d29ea2995e0372a70051be452d Mon Sep 17 00:00:00 2001
From: Masahiko Sawada <sawada.mshk@gmail.com>
Date: Mon, 10 May 2021 20:32:25 +0900
Subject: [PATCH v37 3/9] Support two-phase commit for foreign transactions.

Co-authored-by: Masahiko Sawada, Ashutosh Bapat
---
 .../postgres_fdw/expected/postgres_fdw.out    |    2 +-
 src/backend/access/rmgrdesc/Makefile          |    1 +
 src/backend/access/rmgrdesc/fdwxactdesc.c     |   59 +
 src/backend/access/rmgrdesc/xlogdesc.c        |    4 +-
 src/backend/access/transam/Makefile           |    2 +
 src/backend/access/transam/fdwxact.c          | 1945 ++++++++++++++++-
 src/backend/access/transam/fdwxact_launcher.c |  587 +++++
 src/backend/access/transam/fdwxact_resolver.c |  339 +++
 src/backend/access/transam/rmgr.c             |    1 +
 src/backend/access/transam/twophase.c         |   35 +
 src/backend/access/transam/xact.c             |    4 +-
 src/backend/access/transam/xlog.c             |   41 +-
 src/backend/catalog/dependency.c              |    5 +-
 src/backend/catalog/system_views.sql          |    3 +
 src/backend/commands/dbcommands.c             |   14 +
 src/backend/commands/foreigncmds.c            |   78 +-
 src/backend/foreign/foreign.c                 |    6 +
 src/backend/nodes/copyfuncs.c                 |   15 +
 src/backend/nodes/equalfuncs.c                |   14 +
 src/backend/parser/gram.y                     |   29 +-
 src/backend/postmaster/bgworker.c             |    8 +
 src/backend/postmaster/postmaster.c           |   13 +-
 src/backend/replication/logical/decode.c      |    1 +
 src/backend/replication/syncrep.c             |   14 +-
 src/backend/storage/ipc/ipci.c                |    6 +
 src/backend/storage/ipc/procarray.c           |   42 +-
 src/backend/storage/lmgr/lwlocknames.txt      |    2 +
 src/backend/tcop/postgres.c                   |   16 +
 src/backend/tcop/utility.c                    |   12 +-
 src/backend/utils/activity/wait_event.c       |   15 +
 src/backend/utils/misc/guc.c                  |   48 +
 src/backend/utils/misc/postgresql.conf.sample |   14 +
 src/bin/initdb/initdb.c                       |    1 +
 src/bin/pg_controldata/pg_controldata.c       |    2 +
 src/bin/pg_resetwal/pg_resetwal.c             |    2 +
 src/bin/pg_waldump/rmgrdesc.c                 |    1 +
 src/include/access/fdwxact.h                  |   88 +-
 src/include/access/fdwxact_launcher.h         |   29 +
 src/include/access/fdwxact_resolver.h         |   22 +
 src/include/access/fdwxact_xlog.h             |   49 +
 src/include/access/resolver_internal.h        |   59 +
 src/include/access/rmgrlist.h                 |    1 +
 src/include/access/twophase.h                 |    1 +
 src/include/access/xlog_internal.h            |    1 +
 src/include/catalog/pg_control.h              |    1 +
 src/include/catalog/pg_proc.dat               |   23 +
 src/include/commands/defrem.h                 |    2 +
 src/include/foreign/fdwapi.h                  |    2 +
 src/include/nodes/nodes.h                     |    1 +
 src/include/nodes/parsenodes.h                |   10 +-
 src/include/replication/syncrep.h             |    2 +-
 src/include/storage/procarray.h               |    1 +
 src/include/utils/guc_tables.h                |    2 +
 src/include/utils/wait_event.h                |    7 +-
 src/test/regress/expected/rules.out           |    8 +
 55 files changed, 3623 insertions(+), 67 deletions(-)
 create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c
 create mode 100644 src/backend/access/transam/fdwxact_launcher.c
 create mode 100644 src/backend/access/transam/fdwxact_resolver.c
 create mode 100644 src/include/access/fdwxact_launcher.h
 create mode 100644 src/include/access/fdwxact_resolver.h
 create mode 100644 src/include/access/fdwxact_xlog.h
 create mode 100644 src/include/access/resolver_internal.h

diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out
index 343b33473b..01c43d80ff 100644
--- a/contrib/postgres_fdw/expected/postgres_fdw.out
+++ b/contrib/postgres_fdw/expected/postgres_fdw.out
@@ -9309,7 +9309,7 @@ SELECT count(*) FROM ft1;
 
 -- error here
 PREPARE TRANSACTION 'fdw_tpc';
-ERROR:  cannot PREPARE a transaction that has operated on foreign tables
+ERROR:  cannot PREPARE a distributed transaction that has operated on a foreign server not supporting two-phase commit protocol
 ROLLBACK;
 WARNING:  there is no transaction in progress
 -- ===================================================================
diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile
index f88d72fd86..982c1a36cc 100644
--- a/src/backend/access/rmgrdesc/Makefile
+++ b/src/backend/access/rmgrdesc/Makefile
@@ -13,6 +13,7 @@ OBJS = \
 	clogdesc.o \
 	committsdesc.o \
 	dbasedesc.o \
+	fdwxactdesc.o \
 	genericdesc.o \
 	gindesc.o \
 	gistdesc.o \
diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c
new file mode 100644
index 0000000000..4e97486640
--- /dev/null
+++ b/src/backend/access/rmgrdesc/fdwxactdesc.c
@@ -0,0 +1,59 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxactdesc.c
+ *		rmgr descriptor routines for access/transam/fdwxact.c
+ *
+ * This module describes the WAL records for foreign transaction manager.
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/backend/access/rmgrdesc/fdwxactdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/fdwxact_xlog.h"
+
+void
+fdwxact_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		FdwXactStateOnDiskData *fdwxact_insert = (FdwXactStateOnDiskData *) rec;
+
+		appendStringInfo(buf, "xid: %u, dbid: %u, umid: %u, serverid: %u, owner: %u, identifier: %s",
+						 fdwxact_insert->xid,
+						 fdwxact_insert->dbid,
+						 fdwxact_insert->umid,
+						 fdwxact_insert->serverid,
+						 fdwxact_insert->owner,
+						 fdwxact_insert->identifier);
+	}
+	else
+	{
+		xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec;
+
+		appendStringInfo(buf, "xid: %u, umid: %u",
+						 fdwxact_remove->xid,
+						 fdwxact_remove->umid);
+	}
+
+}
+
+const char *
+fdwxact_identify(uint8 info)
+{
+	switch (info & ~XLR_INFO_MASK)
+	{
+		case XLOG_FDWXACT_INSERT:
+			return "NEW FOREIGN TRANSACTION";
+		case XLOG_FDWXACT_REMOVE:
+			return "REMOVE FOREIGN TRANSACTION";
+	}
+	/* Keep compiler happy */
+	return NULL;
+}
diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c
index e6090a9dad..6bd3bb7700 100644
--- a/src/backend/access/rmgrdesc/xlogdesc.c
+++ b/src/backend/access/rmgrdesc/xlogdesc.c
@@ -112,12 +112,14 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 
 		appendStringInfo(buf, "max_connections=%d max_worker_processes=%d "
 						 "max_wal_senders=%d max_prepared_xacts=%d "
+						 "max_prepared_foreign_transactions=%d"
 						 "max_locks_per_xact=%d wal_level=%s "
-						 "wal_log_hints=%s track_commit_timestamp=%s",
+						 "wal_log_hints=%s track_commit_timestamp=%s ",
 						 xlrec.MaxConnections,
 						 xlrec.max_worker_processes,
 						 xlrec.max_wal_senders,
 						 xlrec.max_prepared_xacts,
+						 xlrec.max_prepared_foreign_xacts,
 						 xlrec.max_locks_per_xact,
 						 wal_level_str,
 						 xlrec.wal_log_hints ? "on" : "off",
diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile
index b05a88549d..26a5ee589c 100644
--- a/src/backend/access/transam/Makefile
+++ b/src/backend/access/transam/Makefile
@@ -16,6 +16,8 @@ OBJS = \
 	clog.o \
 	commit_ts.o \
 	fdwxact.o \
+	fdwxact_launcher.o \
+	fdwxact_resolver.o \
 	generic_xlog.o \
 	multixact.o \
 	parallel.o \
diff --git a/src/backend/access/transam/fdwxact.c b/src/backend/access/transam/fdwxact.c
index ae3fdbdf83..0c6e80a6de 100644
--- a/src/backend/access/transam/fdwxact.c
+++ b/src/backend/access/transam/fdwxact.c
@@ -13,6 +13,57 @@
  * transaction manager calls corresponding FDW API to end the foreign
  * tranasctions.
  *
+ * To achieve commit among all foreign servers atomically, the global transaction
+ * manager supports two-phase commit protocol, which is a type of atomic commitment
+ * protocol. We WAL log the foreign transaction state so foreign transaction state
+ * is crash-safe.
+ *
+ * FOREIGN TRANSACTION RESOLUTION
+ *
+ * At PREPARE TRANSACTION, we prepare all transactions on foreign servers by executing
+ * PrepareForeignTransaction() API for each foreign transaction regardless of data on
+ * the foreign server having been modified.  At COMMIT PREPARED and ROLLBACK PREPARED,
+ * we commit or rollback only the local transaction but not do anything for involved
+ * foreign transactions.  The prepared foreign transactinos are resolved by a resolver
+ * process asynchronously.  Also, users can use pg_resolve_foreign_xact() SQL function
+ * that resolve a foreign transaction manually.
+ *
+ * LOCKING
+ *
+ * Whenever a foreign transaction is processed, the corresponding FdwXactState
+ * entry is updated. To avoid holding the lock during transaction processing
+ * which may take an unpredictable time the in-memory data of foreign
+ * transaction follows a locking model based on the following linked concepts:
+ *
+ * * A process who is going to work on the foreign transaction needs to set
+ *	 locking_backend of the FdwXactState entry, which prevents the entry from being
+ *	 updated and removed by concurrent processes.
+ * * All FdwXactState fields except for status are protected by FdwXactLock.  The
+ *   status is protected by its mutex.
+ *
+ * RECOVERY
+ *
+ * During replay WAL and replication FdwXactCtl also holds information about
+ * active prepared foreign transaction that haven't been moved to disk yet.
+ *
+ * Replay of fdwxact records happens by the following rules:
+ *
+ * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXactState
+ *	 with entries marked with fdwxact->inredo and fdwxact->ondisk.	FdwXactState file
+ *	 data older than the XID horizon of the redo position are discarded.
+ * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->xacts.
+ *	 We set fdwxact->inredo to true for such entries.
+ * * On Checkpoint we iterate through FdwXactCtl->xacts entries that
+ *	 have fdwxact->inredo set and are behind the redo_horizon.	We save
+ *	 them to disk and then set fdwxact->ondisk to true.
+ * * On resolution we delete the entry from FdwXactCtl->xacts.  If
+ *	 fdwxact->ondisk is true, the corresponding entry from the disk is
+ *	 additionally deleted.
+ * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through
+ *	 fdwxact->inredo entries that have not made it to disk.
+ *
+ * These replay rules are borrowed from twophase.c
+ *
  * Portions Copyright (c) 2021, PostgreSQL Global Development Group
  *
  * IDENTIFICATION
@@ -21,15 +72,42 @@
  */
 #include "postgres.h"
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
 #include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "access/xact.h"
 #include "access/xlog.h"
+#include "access/twophase.h"
+#include "access/resolver_internal.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "access/xlogutils.h"
 #include "catalog/pg_user_mapping.h"
 #include "foreign/fdwapi.h"
 #include "foreign/foreign.h"
+#include "funcapi.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "replication/syncrep.h"
+#include "storage/ipc.h"
+#include "storage/latch.h"
+#include "storage/lock.h"
+#include "storage/procarray.h"
+#include "storage/sinvaladt.h"
+#include "utils/builtins.h"
+#include "utils/guc.h"
 #include "utils/memutils.h"
+#include "utils/rel.h"
 #include "utils/syscache.h"
 
+/* Directory where the foreign prepared transaction files will reside */
+#define FDWXACTS_DIR "pg_fdwxact"
+
 /* Initial size of the hash table */
 #define FDWXACT_HASH_SIZE	64
 
@@ -37,6 +115,23 @@
 #define ServerSupportTransactionCallback(fdwent) \
 	(((FdwXactEntry *)(fdwent))->commit_foreign_xact_fn != NULL)
 
+/* Check the FdwXactEntry is capable of two-phase commit  */
+#define ServerSupportTwophaseCommit(fdwent) \
+	(((FdwXactEntry *)(fdwent))->prepare_foreign_xact_fn != NULL)
+
+/*
+ * Name of foreign prepared transaction file is 8 bytes xid and
+ * user mapping OID separated by '_'.
+ *
+ * Since FdwXactState is identified by user mapping OID and it's unique
+ * within a distributed transaction, the name is fairly enough to
+ * ensure uniqueness.
+ */
+#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8)
+#define FdwXactStateFilePath(path, xid, umid)	\
+	snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X", \
+			 xid, umid)
+
 /*
  * Structure to bundle the foreign transaction participant.
  *
@@ -51,26 +146,146 @@ typedef struct FdwXactEntry
 	ForeignServer *server;
 	UserMapping *usermapping;
 
+	/*
+	 * Pointer to a FdwXactState entry in the global array. NULL if the entry
+	 * is not inserted yet but this is registered as a participant.
+	 */
+	FdwXactState fdwxact;
+
 	/* Callbacks for foreign transaction */
 	CommitForeignTransaction_function commit_foreign_xact_fn;
 	RollbackForeignTransaction_function rollback_foreign_xact_fn;
+	PrepareForeignTransaction_function prepare_foreign_xact_fn;
 } FdwXactEntry;
 
 /*
- * Foreign transaction participants involved in the current transaction.
- * A member of participants must support both commit and rollback APIs
+ * The current distributed transaction state.  Members of participants
+ * must support at least both commit and rollback APIs
  * (i.g., ServerSupportTransactionCallback() is true).
  */
-static HTAB *FdwXactParticipants = NULL;
+typedef struct DistributedXactStateData
+{
+	bool		local_prepared; /* will (did) we prepare the local transaction? */
+
+	/* Statistics of participants */
+	int			nparticipants_no_twophase;	/* how many participants doesn't
+											 * support two-phase commit
+											 * protocol? */
+
+	HTAB	   *participants;	/* foreign transaction participants (FdwXactEntry) */
+	List	   *serveroids_uniq;	/* list of unique server OIDs in
+									 * participants */
+} DistributedXactStateData;
+static DistributedXactStateData DistributedXactState = {
+	.local_prepared = false,
+	.nparticipants_no_twophase = 0,
+	.participants = NULL,
+	.serveroids_uniq = NIL,
+};
 
 /* Check the current transaction has at least one fdwxact participant */
 #define HasFdwXactParticipant() \
-	(FdwXactParticipants != NULL && \
-	 hash_get_num_entries(FdwXactParticipants) > 0)
+	(DistributedXactState.participants != NULL && \
+	 hash_get_num_entries(DistributedXactState.participants) > 0)
+
+/* Keep track of registering process exit call back. */
+static bool fdwXactExitRegistered = false;
+
+/* Guc parameter */
+int			max_prepared_foreign_xacts = 0;
+int			max_foreign_xact_resolvers = 0;
 
+static void RemoveFdwXactEntry(Oid umid);
 static void EndFdwXactEntry(FdwXactEntry *fdwent, bool isCommit,
 							bool is_parallel_worker);
-static void RemoveFdwXactEntry(Oid umid);
+static char *getFdwXactIdentifier(FdwXactEntry *fdwent, TransactionId xid);
+static void ForgetAllParticipants(void);
+static void FdwXactLaunchResolvers(void);
+
+static void PrepareAllFdwXacts(TransactionId xid);
+static XLogRecPtr FdwXactInsertEntry(TransactionId xid, FdwXactEntry *fdwent,
+									 char *identifier);
+static void AtProcExit_FdwXact(int code, Datum arg);
+static void FdwXactComputeRequiredXmin(void);
+static FdwXactStatus FdwXactGetTransactionFate(TransactionId xid);
+static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn);
+static void FdwXactRedoRemove(TransactionId xid, Oid umid, bool givewarning);
+static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len);
+static char *ProcessFdwXactBuffer(TransactionId xid, Oid umid,
+								  XLogRecPtr insert_start_lsn, bool fromdisk);
+static char *ReadFdwXactStateFile(TransactionId xid, Oid umid);
+static void RemoveFdwXactStateFile(TransactionId xid, Oid umid, bool giveWarning);
+static void RecreateFdwXactFile(TransactionId xid, Oid umid, void *content, int len);
+
+static FdwXactState insert_fdwxact(Oid dbid, TransactionId xid, Oid umid, Oid serverid,
+								   Oid owner, char *identifier);
+static void remove_fdwxact(FdwXactState fdwxact);
+static List *find_fdwxacts(TransactionId xid, Oid umid, Oid dbid);
+static FdwXactState get_fdwxact_with_check(TransactionId xid, Oid umid,
+										   bool check_two_phase);
+static void pg_foreign_xact_callback(int code, Datum arg);
+
+/*
+ * Calculates the size of shared memory allocated for maintaining foreign
+ * prepared transaction entries.
+ */
+Size
+FdwXactShmemSize(void)
+{
+	Size		size;
+
+	/* Size for foreign transaction information array */
+	size = offsetof(FdwXactCtlData, xacts);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactState)));
+	size = MAXALIGN(size);
+	size = add_size(size, mul_size(max_prepared_foreign_xacts,
+								   sizeof(FdwXactStateData)));
+
+	return size;
+}
+
+/*
+ * Initialization of shared memory for maintaining foreign prepared transaction
+ * entries. The shared memory layout is defined in definition of FdwXactCtlData
+ * structure.
+ */
+void
+FdwXactShmemInit(void)
+{
+	bool		found;
+
+	FdwXactCtl = ShmemInitStruct("Foreign transactions table",
+								 FdwXactShmemSize(),
+								 &found);
+	if (!IsUnderPostmaster)
+	{
+		FdwXactState fdwxacts;
+		int			cnt;
+
+		Assert(!found);
+		FdwXactCtl->free_fdwxacts = NULL;
+		FdwXactCtl->num_xacts = 0;
+
+		/* Initialize the linked list of free FDW transactions */
+		fdwxacts = (FdwXactState)
+			((char *) FdwXactCtl +
+			 MAXALIGN(offsetof(FdwXactCtlData, xacts) +
+					  sizeof(FdwXactState) * max_prepared_foreign_xacts));
+		for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++)
+		{
+			fdwxacts[cnt].status = FDWXACT_STATUS_INVALID;
+			fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+			FdwXactCtl->free_fdwxacts = &fdwxacts[cnt];
+			SpinLockInit(&fdwxacts[cnt].mutex);
+		}
+	}
+	else
+	{
+		Assert(FdwXactCtl);
+		Assert(found);
+	}
+}
 
 /*
  * Register the given foreign transaction participant identified by the
@@ -87,20 +302,21 @@ FdwXactRegisterEntry(UserMapping *usermapping)
 
 	Assert(IsTransactionState());
 
-	if (FdwXactParticipants == NULL)
+	if (DistributedXactState.participants == NULL)
 	{
 		HASHCTL		ctl;
 
 		ctl.keysize = sizeof(Oid);
 		ctl.entrysize = sizeof(FdwXactEntry);
 
-		FdwXactParticipants = hash_create("fdw xact participants",
-										  FDWXACT_HASH_SIZE,
-										  &ctl, HASH_ELEM | HASH_BLOBS);
+		DistributedXactState.participants = hash_create("fdw xact participants",
+														FDWXACT_HASH_SIZE,
+														&ctl, HASH_ELEM | HASH_BLOBS);
 	}
 
 	umid = usermapping->umid;
-	fdwent = hash_search(FdwXactParticipants, (void *) &umid, HASH_ENTER, &found);
+	fdwent = hash_search(DistributedXactState.participants,
+						 (void *) &umid, HASH_ENTER, &found);
 
 	if (found)
 		return;
@@ -124,13 +340,22 @@ FdwXactRegisterEntry(UserMapping *usermapping)
 		ereport(ERROR,
 				(errmsg("cannot register foreign server not supporting transaction callback")));
 
+	fdwent->fdwxact = NULL;
 	fdwent->commit_foreign_xact_fn = routine->CommitForeignTransaction;
 	fdwent->rollback_foreign_xact_fn = routine->RollbackForeignTransaction;
+	fdwent->prepare_foreign_xact_fn = routine->PrepareForeignTransaction;
 
 	MemoryContextSwitchTo(old_ctx);
+
+	/* Update statistics */
+	if (!ServerSupportTwophaseCommit(fdwent))
+		DistributedXactState.nparticipants_no_twophase++;
+
+	Assert(DistributedXactState.nparticipants_no_twophase <=
+		   hash_get_num_entries(DistributedXactState.participants));
 }
 
-/* Remove the foreign transaction from FdwXactParticipants */
+/* Remove the foreign transaction from the current participants */
 void
 FdwXactUnregisterEntry(UserMapping *usermapping)
 {
@@ -145,7 +370,21 @@ FdwXactUnregisterEntry(UserMapping *usermapping)
 static void
 RemoveFdwXactEntry(Oid umid)
 {
-	(void) hash_search(FdwXactParticipants, (void *) &umid, HASH_REMOVE, NULL);
+	FdwXactEntry *fdwent;
+
+	Assert(DistributedXactState.participants != NULL);
+	fdwent = hash_search(DistributedXactState.participants, (void *) &umid,
+						 HASH_REMOVE, NULL);
+
+	if (fdwent)
+	{
+		/* Update statistics */
+		if (!ServerSupportTwophaseCommit(fdwent))
+			DistributedXactState.nparticipants_no_twophase--;
+
+		Assert(DistributedXactState.nparticipants_no_twophase <=
+			   hash_get_num_entries(DistributedXactState.participants));
+	}
 }
 
 /*
@@ -154,29 +393,75 @@ RemoveFdwXactEntry(Oid umid)
 void
 AtEOXact_FdwXact(bool isCommit, bool is_parallel_worker)
 {
-	FdwXactEntry *fdwent;
-	HASH_SEQ_STATUS scan;
-
 	/* If there are no foreign servers involved, we have no business here */
 	if (!HasFdwXactParticipant())
 		return;
 
-	hash_seq_init(&scan, FdwXactParticipants);
-	while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan)))
+	Assert(!RecoveryInProgress());
+
+	if (!isCommit)
 	{
-		Assert(ServerSupportTransactionCallback(fdwent));
+		HASH_SEQ_STATUS scan;
+		FdwXactEntry *fdwent;
 
-		/* Commit or rollback foreign transaction */
-		EndFdwXactEntry(fdwent, isCommit, is_parallel_worker);
+		/* Rollback foreign transactions in the participant list */
+		hash_seq_init(&scan, DistributedXactState.participants);
+		while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan)))
+		{
+			FdwXactState fdwxact = fdwent->fdwxact;
+			int			status;
 
-		/*
-		 * Remove the entry so that we don't recursively process this foreign
-		 * transaction.
-		 */
-		RemoveFdwXactEntry(fdwent->umid);
+			/*
+			 * If this foreign transaction is not prepared yet, end the
+			 * foreign transaction in one-phase.
+			 */
+			if (!fdwxact)
+			{
+				Assert(ServerSupportTransactionCallback(fdwent));
+				EndFdwXactEntry(fdwent, false, is_parallel_worker);
+
+				/*
+				 * Remove FdwXactState entry to prevent processing again in a
+				 * recursive error case.
+				 */
+				RemoveFdwXactEntry(fdwent->umid);
+				continue;
+			}
+
+			/*
+			 * If the foreign transaction has FdwXactState entry, the foreign
+			 * transaction might have been prepared.  We rollback the foreign
+			 * transaction anyway to end the current transaction if the status
+			 * is in-progress.  Since the transaction might have been already
+			 * prepared on the foreign we set the status to aborting and leave
+			 * it.
+			 */
+			SpinLockAcquire(&(fdwxact->mutex));
+			status = fdwxact->status;
+			fdwxact->status = FDWXACT_STATUS_ABORTING;
+			SpinLockRelease(&(fdwxact->mutex));
+
+			if (status == FDWXACT_STATUS_PREPARING)
+				EndFdwXactEntry(fdwent, isCommit, is_parallel_worker);
+		}
 	}
 
-	Assert(!HasFdwXactParticipant());
+	/* Unlock all participants */
+	ForgetAllParticipants();
+
+	/*
+	 * Launch the resolver processes if we failed to prepare the local
+	 * transaction after preparing the foreign transactions.  In this case, we
+	 * need to rollback the prepared transaction on the foreign servers.
+	 */
+	if (DistributedXactState.local_prepared && !isCommit)
+		FdwXactLaunchResolvers();
+
+	/* Reset all fields */
+	DistributedXactState.local_prepared = false;
+	DistributedXactState.nparticipants_no_twophase = 0;
+	list_free(DistributedXactState.serveroids_uniq);
+	DistributedXactState.serveroids_uniq = NIL;
 }
 
 /*
@@ -193,6 +478,7 @@ EndFdwXactEntry(FdwXactEntry *fdwent, bool isCommit, bool is_parallel_worker)
 	finfo.usermapping = fdwent->usermapping;
 	finfo.flags = FDWXACT_FLAG_ONEPHASE |
 		((is_parallel_worker) ? FDWXACT_FLAG_PARALLEL_WORKER : 0);
+	finfo.identifier = NULL;
 
 	if (isCommit)
 	{
@@ -209,15 +495,1610 @@ EndFdwXactEntry(FdwXactEntry *fdwent, bool isCommit, bool is_parallel_worker)
 }
 
 /*
- * This function is called at PREPARE TRANSACTION.  Since we don't support
- * preparing foreign transactions for now, raise an error if the local transaction
- * has any foreign transaction.
+ * Prepare foreign transactions by PREPARE TRANSACTION command.
+ *
+ * In case where an error happens during parparing a foreign transaction we
+ * change to rollback.  See AtEOXact_FdwXact() for details.
  */
 void
 AtPrepare_FdwXact(void)
 {
-	if (HasFdwXactParticipant())
+	TransactionId xid;
+
+	/* If there are no foreign servers involved, we have no business here */
+	if (!HasFdwXactParticipant())
+		return;
+
+	/*
+	 * Check if there is a server that doesn't support two-phase commit. All
+	 * involved servers need to support two-phase commit as we're going to
+	 * prepare all of them.
+	 */
+	if (DistributedXactState.nparticipants_no_twophase > 0)
 		ereport(ERROR,
 				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
-				 errmsg("cannot PREPARE a transaction that has operated on foreign tables")));
+				 errmsg("cannot PREPARE a distributed transaction that has operated on a foreign server not supporting two-phase commit protocol")));
+
+	/*
+	 * Assign a transaction id if not yet because the local transaction id
+	 * is needed to determine the result of the distributed transaction.
+	 */
+	xid = GetTopTransactionId();
+
+	/*
+	 * Mark the local transaction will be prepared before actually preparing
+	 * any foreign trasactions so that in a case where an error happens during
+	 * preparing a foreign transaction or preparing the local transaction, we can
+	 * launch a resolver to rollback already-prepared foreign transactions.
+	 */
+	DistributedXactState.local_prepared = true;
+
+	PrepareAllFdwXacts(xid);
+}
+
+/*
+ * Pre-commit processing for foreign transactions. We commit those foreign
+ * transactions with one-phase.
+ */
+void
+PreCommit_FdwXact(bool is_parallel_worker)
+{
+	HASH_SEQ_STATUS scan;
+	FdwXactEntry *fdwent;
+
+	/*
+	 * If there is no foreign server involved or all foreign transactions are
+	 * already prepared (see AtPrepare_FdwXact()), we have no business here.
+	 */
+	if (!HasFdwXactParticipant() || DistributedXactState.local_prepared)
+		return;
+
+	Assert(!RecoveryInProgress());
+
+	/* Commit all foreign transactions in the participant list */
+	hash_seq_init(&scan, DistributedXactState.participants);
+	while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan)))
+	{
+		Assert(ServerSupportTransactionCallback(fdwent));
+
+		/*
+		 * Commit the foreign transaction and remove itself from the hash
+		 * table so that we don't try to abort already-closed transaction.
+		 */
+		EndFdwXactEntry(fdwent, true, is_parallel_worker);
+		RemoveFdwXactEntry(fdwent->umid);
+	}
+}
+
+/*
+ * Functions to count the number of FdwXactState entries associated with
+ * the given argument.
+ */
+int
+CountFdwXactsForUserMapping(Oid umid)
+{
+	List	   *res;
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	res = find_fdwxacts(InvalidTransactionId, umid, InvalidOid);
+	LWLockRelease(FdwXactLock);
+
+	return list_length(res);
+}
+
+int
+CountFdwXactsForDB(Oid dbid)
+{
+	List	   *res;
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	res = find_fdwxacts(InvalidTransactionId, InvalidOid, dbid);
+	LWLockRelease(FdwXactLock);
+
+	return list_length(res);
+
+}
+
+/*
+ * We must fsync the foreign transaction state file that is valid or generated
+ * during redo and has a inserted LSN <= the checkpoint's redo horizon.
+ * The foreign transaction entries and hence the corresponding files are expected
+ * to be very short-lived. By executing this function at the end, we might have
+ * lesser files to fsync, thus reducing some I/O. This is similar to
+ * CheckPointTwoPhase().
+ *
+ * This is deliberately run as late as possible in the checkpoint sequence,
+ * because FdwXacts ordinarily have short lifespans, and so it is quite
+ * possible that FdwXactStates that were valid at checkpoint start will no longer
+ * exist if we wait a little bit. With typical checkpoint settings this
+ * will be about 3 minutes for an online checkpoint, so as a result we
+ * expect that there will be no FdwXactStates that need to be copied to disk.
+ *
+ * If a FdwXactState remains valid across multiple checkpoints, it will already
+ * be on disk so we don't bother to repeat that write.
+ */
+void
+CheckPointFdwXacts(XLogRecPtr redo_horizon)
+{
+	int			cnt;
+	int			serialized_fdwxacts = 0;
+
+	if (max_prepared_foreign_xacts == 0)
+		return;					/* nothing to do */
+
+	/*
+	 * We are expecting there to be zero FdwXactState that need to be copied
+	 * to disk, so we perform all I/O while holding FdwXactLock for
+	 * simplicity. This presents any new foreign xacts from preparing while
+	 * this occurs, which shouldn't be a problem since the presence of
+	 * long-lived prepared foreign xacts indicated the transaction manager
+	 * isn't active.
+	 *
+	 * It's also possible to move I/O out of the lock, but on every error we
+	 * should check whether somebody committed our transaction in different
+	 * backend. Let's leave this optimisation for future, if somebody will
+	 * spot that this place cause bottleneck.
+	 *
+	 * Note that it isn't possible for there to be a FdwXactState with a
+	 * insert_end_lsn set prior to the last checkpoint yet is marked invalid,
+	 * because of the efforts with delayChkpt.
+	 */
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (cnt = 0; cnt < FdwXactCtl->num_xacts; cnt++)
+	{
+		FdwXactState fdwxact = FdwXactCtl->xacts[cnt];
+
+		if ((fdwxact->valid || fdwxact->inredo) &&
+			!fdwxact->ondisk &&
+			fdwxact->insert_end_lsn <= redo_horizon)
+		{
+			char	   *buf;
+			int			len;
+
+			XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len);
+			RecreateFdwXactFile(fdwxact->data.xid, fdwxact->data.umid, buf, len);
+			fdwxact->ondisk = true;
+			fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+			fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+			pfree(buf);
+			serialized_fdwxacts++;
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Flush unconditionally the parent directory to make any information
+	 * durable on disk.	 FdwXactState files could have been removed and those
+	 * removals need to be made persistent as well as any files newly created.
+	 */
+	fsync_fname(FDWXACTS_DIR, true);
+
+	if (log_checkpoints && serialized_fdwxacts > 0)
+		ereport(LOG,
+				(errmsg_plural("%u foreign transaction state file was written "
+							   "for long-running prepared transactions",
+							   "%u foreign transaction state files were written "
+							   "for long-running prepared transactions",
+							   serialized_fdwxacts,
+							   serialized_fdwxacts)));
+}
+
+/*
+ * Prepare all foreign transactions.
+ *
+ * The basic strategy is to create all FdwXactState entries with WAL-logging,
+ * wait for those WAL records to be replicated if synchronous replication is
+ * enabled, and then prepare foreign transactions by calling
+ * PrepareForeignTransaction FDW callback functions.  There are two points
+ * here: (a) writing WAL records before preparing foreign transactions, and
+ * (b) waiting for those records to replicated before preparing foreign
+ * transactions.
+ *
+ * You might think that we can itereate over foreign transaction while preparing
+ * the foreign transaction and writing WAL record one by one.  But this doesn't
+ * work if the server crashes in the middle of those operations. We will end
+ * up losing foreign prepared transaction information if the server crashes
+ * and/or the failover happens.  Therefore, we need point (a) because otherwise
+ * we will lost the prepared transaction on the foreign server and will not be
+ * able to resolve it after the server crash.  Hence  persist first then prepare.
+ * Point (b) guarantees that foreign transaction information are not lost even
+ * if the failover happens.
+ */
+static void
+PrepareAllFdwXacts(TransactionId xid)
+{
+	FdwXactEntry *fdwent;
+	XLogRecPtr	flush_lsn;
+	bool	canceled;
+	HASH_SEQ_STATUS scan;
+
+	Assert(TransactionIdIsValid(xid));
+
+	/* Persist all foreign transaction entries */
+	hash_seq_init(&scan, DistributedXactState.participants);
+	while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan)))
+	{
+		char	   *identifier;
+
+		Assert(ServerSupportTwophaseCommit(fdwent));
+
+		CHECK_FOR_INTERRUPTS();
+
+		/* Get prepared transaction identifier */
+		identifier = getFdwXactIdentifier(fdwent, xid);
+		Assert(identifier);
+
+		/*
+		 * Insert the foreign transaction entry with the
+		 * FDWXACT_STATUS_PREPARING status. Registration persists this
+		 * information to the disk and logs (that way relaying it on standby).
+		 * Thus in case we loose connectivity to the foreign server or crash
+		 * ourselves, we will remember that we might have prepared transaction
+		 * on the foreign server and try to resolve it when connectivity is
+		 * restored or after crash recovery.
+		 */
+		flush_lsn = FdwXactInsertEntry(xid, fdwent, identifier);
+	}
+
+	HOLD_INTERRUPTS();
+
+	/* Wait for all WAL records to be replicated, if necessary */
+	canceled = SyncRepWaitForLSN(flush_lsn, false);
+
+	RESUME_INTERRUPTS();
+
+	/*
+	 * XXX: dirty hack to invoke an interruption that was absorbed.
+	 * SyncRepWaitForLSN() is aimed to be used at the point where we never be
+	 * able to change to rollback.  But at this state, preparing foreign
+	 * transactions at pre-commit phase, we still are able to change to rollback.
+	 * So if the waits canceled by an interruption, we error out. Note that
+	 * SyncRepWaitForLSN() might have been set ProcDiePending too.
+	 */
+	if (canceled)
+	{
+		QueryCancelPending = true;
+		ProcessInterrupts();
+	}
+
+	/* Prepare all foreign transactions */
+	hash_seq_init(&scan, DistributedXactState.participants);
+	while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan)))
+	{
+		FdwXactInfo finfo;
+		FdwXactState fdwxact = fdwent->fdwxact;
+
+		/*
+		 * Prepare the foreign transaction.  Between FdwXactInsertEntry call
+		 * till this backend hears acknowledge from foreign server, the
+		 * backend may abort the local transaction (say, because of a signal).
+		 */
+		finfo.server = fdwent->server;
+		finfo.usermapping = fdwent->usermapping;
+		finfo.flags = 0;
+		finfo.identifier = pstrdup(fdwxact->data.identifier);
+		fdwent->prepare_foreign_xact_fn(&finfo);
+
+		/* succeeded, update status */
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwent->fdwxact->status = FDWXACT_STATUS_PREPARED;
+		SpinLockRelease(&fdwxact->mutex);
+
+		pfree(finfo.identifier);
+	}
+}
+
+/*
+ * Return a null-terminated foreign transaction identifier.  We generate an
+ * unique identifier with in the form of
+ * "fx_<random number>_<xid>_<umid> whose length is less than FDWXACT_ID_MAX_LEN.
+ *
+ * Returned string value is used to identify foreign transaction. The
+ * identifier should not be same as any other concurrent prepared transaction
+ * identifier.
+ *
+ * To make the foreign transactionid unique, we should ideally use something
+ * like UUID, which gives unique ids with high probability, but that may be
+ * expensive here and UUID extension which provides the function to generate
+ * UUID is not part of the core code.
+ */
+static char *
+getFdwXactIdentifier(FdwXactEntry *fdwent, TransactionId xid)
+{
+	char		buf[FDWXACT_ID_MAX_LEN] = {0};
+
+	snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%u", Abs(random()),
+			 xid, fdwent->umid);
+
+	return pstrdup(buf);
+}
+
+/*
+ * This function insert a new FdwXactState entry to the global array with
+ * WAL-logging. The new entry is held by the backend who inserted.
+ */
+static XLogRecPtr
+FdwXactInsertEntry(TransactionId xid, FdwXactEntry *fdwent, char *identifier)
+{
+	FdwXactStateOnDiskData *fdwxact_file_data;
+	FdwXactState fdwxact;
+	Oid			owner;
+	int			data_len;
+
+	/* on first call, register the exit hook */
+	if (!fdwXactExitRegistered)
+	{
+		before_shmem_exit(AtProcExit_FdwXact, 0);
+		fdwXactExitRegistered = true;
+	}
+
+	/*
+	 * Enter the foreign transaction into the shared memory structure.
+	 */
+	owner = GetUserId();
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	fdwxact = insert_fdwxact(MyDatabaseId, xid, fdwent->umid,
+							 fdwent->usermapping->serverid, owner, identifier);
+	fdwxact->locking_backend = MyBackendId;
+	LWLockRelease(FdwXactLock);
+
+	/* Update FdwXactEntry */
+	fdwent->fdwxact = fdwxact;
+
+	/* Remember server's oid where we prepared the transaction */
+	DistributedXactState.serveroids_uniq =
+		list_append_unique_oid(DistributedXactState.serveroids_uniq,
+							   fdwent->usermapping->serverid);
+
+	/*
+	 * Prepare to write the entry to a file. Also add xlog entry. The contents
+	 * of the xlog record are same as what is written to the file.
+	 */
+	data_len = offsetof(FdwXactStateOnDiskData, identifier);
+	data_len = data_len + strlen(identifier) + 1;
+	data_len = MAXALIGN(data_len);
+	fdwxact_file_data = (FdwXactStateOnDiskData *) palloc0(data_len);
+	memcpy(fdwxact_file_data, &(fdwxact->data), data_len);
+
+	START_CRIT_SECTION();
+
+	/* See note in RecordTransactionCommit */
+	MyProc->delayChkpt = true;
+
+	/* Add the entry in the xlog and save LSN for checkpointer */
+	XLogBeginInsert();
+	XLogRegisterData((char *) fdwxact_file_data, data_len);
+	fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT);
+
+	/* If we crash now, we have prepared: WAL replay will fix things */
+
+	/* Store record's start location to read that later on CheckPoint */
+	fdwxact->insert_start_lsn = ProcLastRecPtr;
+
+	/* File is written completely, checkpoint can proceed with syncing */
+	fdwxact->valid = true;
+
+	/* Checkpoint can process now */
+	MyProc->delayChkpt = false;
+
+	END_CRIT_SECTION();
+
+	pfree(fdwxact_file_data);
+	return fdwxact->insert_end_lsn;
+}
+
+/*
+ * Insert a new entry for a given foreign transaction identified by transaction
+ * id, foreign server and user mapping, into the shared memory array. Caller
+ * must hold FdwXactLock in exclusive mode.
+ *
+ * If the entry already exists, the function raises an error.
+ */
+static FdwXactState
+insert_fdwxact(Oid dbid, TransactionId xid, Oid umid, Oid serverid, Oid owner,
+			   char *identifier)
+{
+	FdwXactState fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	/* Check for duplicated foreign transaction entry */
+	for (int i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		fdwxact = FdwXactCtl->xacts[i];
+		if (fdwxact->valid &&
+			fdwxact->data.xid == xid &&
+			fdwxact->data.umid == umid)
+			ereport(ERROR,
+					(errmsg("could not insert a foreign transaction entry"),
+					 errdetail("Duplicate entry with transaction id %u, user mapping id %u exists.",
+							   xid, umid)));
+	}
+
+	/*
+	 * Get a next free foreign transaction entry. Raise error if there are
+	 * none left.
+	 */
+	if (!FdwXactCtl->free_fdwxacts)
+	{
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("maximum number of foreign transactions reached"),
+				 errhint("Increase max_prepared_foreign_transactions: \"%d\".",
+						 max_prepared_foreign_xacts)));
+	}
+	fdwxact = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next;
+
+	/* Insert the entry to shared memory array */
+	Assert(FdwXactCtl->num_xacts < max_prepared_foreign_xacts);
+	FdwXactCtl->xacts[FdwXactCtl->num_xacts++] = fdwxact;
+
+	fdwxact->status = FDWXACT_STATUS_PREPARING;
+	fdwxact->data.xid = xid;
+	fdwxact->data.dbid = dbid;
+	fdwxact->data.umid = umid;
+	fdwxact->data.serverid = serverid;
+	fdwxact->data.owner = owner;
+	strlcpy(fdwxact->data.identifier, identifier, FDWXACT_ID_MAX_LEN);
+	fdwxact->data.identifier[strlen(identifier)] = '\0';
+
+	fdwxact->insert_start_lsn = InvalidXLogRecPtr;
+	fdwxact->insert_end_lsn = InvalidXLogRecPtr;
+	fdwxact->locking_backend = InvalidBackendId;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+
+	return fdwxact;
+}
+
+/*
+ * Remove the foreign prepared transaction entry from shared memory.
+ * Caller must hold FdwXactLock in exclusive mode.
+ */
+static void
+remove_fdwxact(FdwXactState fdwxact)
+{
+	int			i;
+
+	Assert(fdwxact != NULL);
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	elog(DEBUG2, "remove fdwxact entry id %s", fdwxact->data.identifier);
+
+	if (!RecoveryInProgress())
+	{
+		xl_fdwxact_remove record;
+		XLogRecPtr	recptr;
+
+		/* Fill up the log record before releasing the entry */
+		record.xid = fdwxact->data.xid;
+		record.umid = fdwxact->data.umid;
+
+		/*
+		 * Now writing FdwXactState data to WAL. We have to set delayChkpt
+		 * here, otherwise a checkpoint starting immediately after the WAL
+		 * record is inserted could complete without fsync'ing our state file.
+		 * (This is essentially the same kind of race condition as the
+		 * COMMIT-to-clog-write case that RecordTransactionCommit uses
+		 * delayChkpt for; see notes there.)
+		 */
+		START_CRIT_SECTION();
+
+		MyProc->delayChkpt = true;
+
+		/*
+		 * Log that we are removing the foreign transaction entry and remove
+		 * the file from the disk as well.
+		 */
+		XLogBeginInsert();
+		XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove));
+		recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE);
+
+		/* Always flush, since we're about to remove the FdwXact state file */
+		XLogFlush(recptr);
+
+		/* Now we can mark ourselves as out of the commit critical section */
+		MyProc->delayChkpt = false;
+
+		END_CRIT_SECTION();
+	}
+
+	/* Search the slot where this entry resided */
+	for (i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		if (FdwXactCtl->xacts[i] == fdwxact)
+			break;
+	}
+
+	Assert(i < FdwXactCtl->num_xacts);
+
+	/* Clean up any files we may have left */
+	if (fdwxact->ondisk)
+		RemoveFdwXactStateFile(fdwxact->data.xid, fdwxact->data.umid, true);
+
+	/* Remove the entry from active array */
+	FdwXactCtl->num_xacts--;
+	FdwXactCtl->xacts[i] = FdwXactCtl->xacts[FdwXactCtl->num_xacts];
+
+	/* Put it back into free list */
+	fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts;
+	FdwXactCtl->free_fdwxacts = fdwxact;
+
+	/* Reset informations */
+	fdwxact->status = FDWXACT_STATUS_INVALID;
+	fdwxact->locking_backend = InvalidBackendId;
+	fdwxact->valid = false;
+	fdwxact->ondisk = false;
+	fdwxact->inredo = false;
+}
+
+/*
+ * When the process exits, unlock all the entries.
+ */
+static void
+AtProcExit_FdwXact(int code, Datum arg)
+{
+	ForgetAllParticipants();
+	FdwXactLaunchResolvers();
+}
+
+/*
+ * Unlock all foreign transaction participants.  If we left foreign transaction,
+ * update the oldest xmin of unresolved transaction to prevent the local
+ * transaction id of such unresolved foreign transaction from begin truncated.
+ * Returns the number of remaining foreign transactions.  We return the unique
+ * list of foreign servers' oids so that the caller request to launch the resolver
+ * using it.
+ */
+static void
+ForgetAllParticipants(void)
+{
+	FdwXactEntry *fdwent;
+	HASH_SEQ_STATUS scan;
+	int			nremaining = 0;
+
+	if (!HasFdwXactParticipant())
+		return;
+
+	hash_seq_init(&scan, DistributedXactState.participants);
+	while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan)))
+	{
+		FdwXactState fdwxact = fdwent->fdwxact;
+
+		if (fdwxact)
+		{
+			Assert(fdwxact->locking_backend == MyBackendId);
+
+			/* Unlock the foreign transaction entry */
+			LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+			fdwxact->locking_backend = InvalidBackendId;
+			LWLockRelease(FdwXactLock);
+
+			nremaining++;
+		}
+
+		/* Remove from the participants list */
+		RemoveFdwXactEntry(fdwent->umid);
+	}
+
+	/*
+	 * If we leave any FdwXactState entries, update the oldest local
+	 * transaction of unresolved distributed transaction.
+	 */
+	if (nremaining > 0)
+		FdwXactComputeRequiredXmin();
+
+	Assert(!HasFdwXactParticipant());
+}
+
+static void
+FdwXactLaunchResolvers(void)
+{
+	if (list_length(DistributedXactState.serveroids_uniq) > 0)
+		LaunchOrWakeupFdwXactResolver(DistributedXactState.serveroids_uniq);
+}
+
+/*
+ * Launch or wake up the resolver to resolve the given transaction.
+ */
+void
+FdwXactLaunchResolversForXid(TransactionId xid)
+{
+	List	   *serveroids = NIL;
+
+	if (max_prepared_foreign_xacts == 0)
+		return;					/* nothing to do */
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		FdwXactState fdwxact = FdwXactCtl->xacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		/* Collect server oids associated with the given transaction */
+		if (fdwxact->data.xid == xid)
+			serveroids = list_append_unique_oid(serveroids, fdwxact->data.serverid);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* Exit if there is no servers to launch */
+	if (serveroids == NIL)
+		return;
+
+	LaunchOrWakeupFdwXactResolver(serveroids);
+	list_free(serveroids);
+}
+
+/*
+ * Commit or rollback one prepared foreign transaction, and remove FdwXactState
+ * entry.
+ */
+void
+ResolveOneFdwXact(FdwXactState fdwxact)
+{
+	FdwXactInfo finfo;
+	FdwRoutine *routine;
+
+	/* The FdwXactState entry must be held by me */
+	Assert(fdwxact != NULL);
+	Assert(fdwxact->locking_backend == MyBackendId);
+	Assert(fdwxact->status == FDWXACT_STATUS_PREPARED ||
+		   fdwxact->status == FDWXACT_STATUS_COMMITTING ||
+		   fdwxact->status == FDWXACT_STATUS_ABORTING);
+
+	/* Set whether we do commit or abort if not set yet */
+	if (fdwxact->status == FDWXACT_STATUS_PREPARED)
+	{
+		FdwXactStatus new_status;
+
+		new_status = FdwXactGetTransactionFate(fdwxact->data.xid);
+		Assert(new_status == FDWXACT_STATUS_COMMITTING ||
+			   new_status == FDWXACT_STATUS_ABORTING);
+
+		/* Update the status */
+		SpinLockAcquire(&fdwxact->mutex);
+		fdwxact->status = new_status;
+		SpinLockRelease(&fdwxact->mutex);
+	}
+
+	routine = GetFdwRoutineByServerId(fdwxact->data.serverid);
+
+	/* Prepare the foreign transaction information to pass to API */
+	finfo.server = GetForeignServer(fdwxact->data.serverid);
+	finfo.usermapping = GetUserMapping(fdwxact->data.owner, fdwxact->data.serverid);
+	finfo.flags = 0;
+	finfo.identifier = fdwxact->data.identifier;
+
+	if (fdwxact->status == FDWXACT_STATUS_COMMITTING)
+	{
+		routine->CommitForeignTransaction(&finfo);
+		elog(DEBUG1, "successfully committed the prepared foreign transaction %s",
+			 fdwxact->data.identifier);
+	}
+	else
+	{
+		routine->RollbackForeignTransaction(&finfo);
+		elog(DEBUG1, "successfully rolled back the prepared foreign transaction %s",
+			 fdwxact->data.identifier);
+	}
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	remove_fdwxact(fdwxact);
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Compute the oldest xmin across all unresolved foreign transactions
+ * and store it in the ProcArray.
+ */
+static void
+FdwXactComputeRequiredXmin(void)
+{
+	TransactionId agg_xmin = InvalidTransactionId;
+
+	Assert(FdwXactCtl != NULL);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+
+	for (int i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		FdwXactState fdwxact = FdwXactCtl->xacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		Assert(TransactionIdIsValid(fdwxact->data.xid));
+
+		/*
+		 * We can exclude entries that are marked as either committing or
+		 * aborting and its state file is on disk since such entries no longer
+		 * need to lookup its transaction status from the commit log.
+		 */
+		if (!TransactionIdIsValid(agg_xmin) ||
+			TransactionIdPrecedes(fdwxact->data.xid, agg_xmin) ||
+			(fdwxact->ondisk &&
+			 (fdwxact->status == FDWXACT_STATUS_COMMITTING ||
+			  fdwxact->status == FDWXACT_STATUS_ABORTING)))
+			agg_xmin = fdwxact->data.xid;
+	}
+
+	LWLockRelease(FdwXactLock);
+
+	ProcArraySetFdwXactUnresolvedXmin(agg_xmin);
+}
+
+
+/*
+ * Return whether the foreign transaction associated with the given transaction
+ * id should be committed or rolled back according to the result of the local
+ * transaction.
+ */
+static FdwXactStatus
+FdwXactGetTransactionFate(TransactionId xid)
+{
+	/*
+	 * If the local transaction is already committed, commit prepared foreign
+	 * transaction.
+	 */
+	if (TransactionIdDidCommit(xid))
+		return FDWXACT_STATUS_COMMITTING;
+
+	/*
+	 * If the local transaction is already aborted, abort prepared foreign
+	 * transactions.
+	 */
+	else if (TransactionIdDidAbort(xid))
+		return FDWXACT_STATUS_ABORTING;
+
+	/*
+	 * The local transaction is not in progress but the foreign transaction is
+	 * not prepared on the foreign server. This can happen when transaction
+	 * failed after registered this entry but before actual preparing on the
+	 * foreign server. So let's assume it aborted.
+	 */
+	else if (!TransactionIdIsInProgress(xid))
+		return FDWXACT_STATUS_ABORTING;
+
+	/*
+	 * The Local transaction is in progress and foreign transaction is about
+	 * to be committed or aborted.	Raise an error anyway since we cannot
+	 * determine the fate of this foreign transaction according to the local
+	 * transaction whose fate is also not determined.
+	 */
+	elog(ERROR,
+		 "cannot resolve the foreign transaction associated with in-process transaction");
+
+	pg_unreachable();
+}
+
+
+/*
+ * Recreates a foreign transaction state file. This is used in WAL replay
+ * and during checkpoint creation.
+ *
+ * Note: content and len don't include CRC.
+ */
+void
+RecreateFdwXactFile(TransactionId xid, Oid umid, void *content, int len)
+{
+	char		path[MAXPGPATH];
+	pg_crc32c	statefile_crc;
+	int			fd;
+
+	/* Recompute CRC */
+	INIT_CRC32C(statefile_crc);
+	COMP_CRC32C(statefile_crc, content, len);
+	FIN_CRC32C(statefile_crc);
+
+	FdwXactStateFilePath(path, xid, umid);
+
+	fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY);
+
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not recreate foreign transaction state file \"%s\": %m",
+						path)));
+
+	/* Write content and CRC */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE);
+	if (write(fd, content, len) != len)
+	{
+		/* if write didn't set errno, assume problem is no disk space */
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c))
+	{
+		if (errno == 0)
+			errno = ENOSPC;
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not write foreign transaction state file: %m")));
+	}
+	pgstat_report_wait_end();
+
+	/*
+	 * We must fsync the file because the end-of-replay checkpoint will not do
+	 * so, there being no FDWXACT in shared memory yet to tell it to.
+	 */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC);
+	if (pg_fsync(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not fsync foreign transaction state file: %m")));
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd) != 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close foreign transaction file: %m")));
+}
+
+
+/* Apply the redo log for a foreign transaction */
+void
+fdwxact_redo(XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == XLOG_FDWXACT_INSERT)
+	{
+		/*
+		 * Add fdwxact entry and set start/end lsn of the WAL record in
+		 * FdwXactState entry.
+		 */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoAdd(XLogRecGetData(record), record->ReadRecPtr,
+					   record->EndRecPtr);
+		LWLockRelease(FdwXactLock);
+	}
+	else if (info == XLOG_FDWXACT_REMOVE)
+	{
+		xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec;
+
+		/* Delete FdwXactState entry and file if exists */
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		FdwXactRedoRemove(record->xid, record->umid, false);
+		LWLockRelease(FdwXactLock);
+	}
+	else
+		elog(ERROR, "invalid log type %d in foreign transaction log record", info);
+
+	return;
+}
+
+/*
+ * Scan the shared memory entries of FdwXactState and determine the range of valid
+ * XIDs present.  This is run during database startup, after we have completed
+ * reading WAL.	 ShmemVariableCache->nextXid has been set to one more than
+ * the highest XID for which evidence exists in WAL.
+
+ * On corrupted two-phase files, fail immediately.	Keeping around broken
+ * entries and let replay continue causes harm on the system, and a new
+ * backup should be rolled in.
+
+ * Our other responsibility is to update and return the oldest valid XID
+ * among the distributed transactions. This is needed to synchronize pg_subtrans
+ * startup properly.
+ */
+TransactionId
+PrescanFdwXacts(TransactionId oldestActiveXid)
+{
+	FullTransactionId nextXid = ShmemVariableCache->nextXid;
+	TransactionId origNextXid = XidFromFullTransactionId(nextXid);
+	TransactionId result = origNextXid;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		FdwXactState fdwxact = FdwXactCtl->xacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->data.xid, fdwxact->data.umid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		if (TransactionIdPrecedes(fdwxact->data.xid, result))
+			result = fdwxact->data.xid;
+
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+
+	return result;
+}
+
+/*
+ * Scan pg_fdwxact and fill FdwXactState depending on the on-disk data.
+ * This is called once at the beginning of recovery, saving any extra
+ * lookups in the future.  FdwXactState files that are newer than the
+ * minimum XID horizon are discarded on the way.
+ */
+void
+RestoreFdwXactData(void)
+{
+	DIR		   *cldir;
+	struct dirent *clde;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	cldir = AllocateDir(FDWXACTS_DIR);
+	while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL)
+	{
+		if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN &&
+			strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN)
+		{
+			TransactionId xid;
+			Oid			umid;
+			char	   *buf;
+
+			sscanf(clde->d_name, "%08x_%08x", &xid, &umid);
+
+			/* Read fdwxact data from disk */
+			buf = ProcessFdwXactBuffer(xid, umid, InvalidXLogRecPtr,
+									   true);
+			if (buf == NULL)
+				continue;
+
+			/* Add this entry into the table of foreign transactions */
+			FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr);
+		}
+	}
+
+	LWLockRelease(FdwXactLock);
+	FreeDir(cldir);
+}
+
+/*
+ * Scan the shared memory entries of FdwXactState and valid them.
+ *
+ * This is run at the end of recovery, but before we allow backends to write
+ * WAL.
+ */
+void
+RecoverFdwXacts(void)
+{
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		FdwXactState fdwxact = FdwXactCtl->xacts[i];
+		char	   *buf;
+
+		buf = ProcessFdwXactBuffer(fdwxact->data.xid, fdwxact->data.umid,
+								   fdwxact->insert_start_lsn, fdwxact->ondisk);
+
+		if (buf == NULL)
+			continue;
+
+		ereport(LOG,
+				(errmsg("recovering foreign prepared transaction %s from shared memory",
+						fdwxact->data.identifier)));
+
+		/* recovered, so reset the flag for entries generated by redo */
+		fdwxact->inredo = false;
+		fdwxact->valid = true;
+		pfree(buf);
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Store pointer to the start/end of the WAL record along with the xid in
+ * a fdwxact entry in shared memory FdwXactData structure.
+ */
+static void
+FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
+{
+	FdwXactStateOnDiskData *fdwxact_data = (FdwXactStateOnDiskData *) buf;
+	FdwXactState fdwxact;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	/*
+	 * Add this entry into the table of foreign transactions. The status of
+	 * the transaction is set as preparing, since we do not know the exact
+	 * status right now. Resolver will set it later based on the status of
+	 * local transaction which prepared this foreign transaction.
+	 */
+	fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->xid,
+							 fdwxact_data->umid, fdwxact_data->serverid,
+							 fdwxact_data->owner, fdwxact_data->identifier);
+
+	elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u user mapping %u owner %u id %s",
+		 fdwxact_data->dbid, fdwxact_data->xid,
+		 fdwxact_data->umid, fdwxact_data->owner,
+		 fdwxact_data->identifier);
+
+	/*
+	 * Set status as PREPARED, since we do not know the xact status right now.
+	 * We will set it later based on the status of local transaction that
+	 * prepared this fdwxact entry.
+	 */
+	fdwxact->status = FDWXACT_STATUS_PREPARED;
+	fdwxact->insert_start_lsn = start_lsn;
+	fdwxact->insert_end_lsn = end_lsn;
+	fdwxact->inredo = true;		/* added in redo */
+	fdwxact->valid = false;
+	fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn);
+}
+
+/*
+ * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove
+ * FdwXactState file if a foreign transaction was saved via an earlier checkpoint.
+ * We could not found the FdwXactState entry in the case where a crash recovery
+ * starts from the point where is after added but before removed the entry.
+ */
+static void
+FdwXactRedoRemove(TransactionId xid, Oid umid, bool givewarning)
+{
+	FdwXactState fdwxact;
+	int			i;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+	Assert(RecoveryInProgress());
+
+	for (i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		fdwxact = FdwXactCtl->xacts[i];
+
+		if (fdwxact->data.xid == xid && fdwxact->data.umid == umid)
+			break;
+	}
+
+	if (i >= FdwXactCtl->num_xacts)
+		return;
+
+	/* Clean up entry and any files we may have left */
+	remove_fdwxact(fdwxact);
+
+	elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction %s",
+		 fdwxact->data.identifier);
+}
+
+/*
+ * Reads foreign transaction data from xlog. During checkpoint this data will
+ * be moved to fdwxact files and ReadFdwXactStateFile should be used instead.
+ *
+ * Note clearly that this function accesses WAL during normal operation, similarly
+ * to the way WALSender or Logical Decoding would do. It does not run during
+ * crash recovery or standby processing.
+ */
+static void
+XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len)
+{
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+	TimeLineID	save_currtli = ThisTimeLineID;
+
+	xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
+									XL_ROUTINE(.page_read = &read_local_xlog_page,
+											   .segment_open = &wal_segment_open,
+											   .segment_close = &wal_segment_close),
+									NULL);
+
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+				 errdetail("Failed while allocating an XLog reading processor.")));
+
+	XLogBeginRead(xlogreader, lsn);
+	record = XLogReadRecord(xlogreader, &errormsg);
+
+	/*
+	 * Restore immediately the timeline where it was previously, as
+	 * read_local_xlog_page() could have changed it if the record was read
+	 * while recovery was finishing or if the timeline has jumped in-between.
+	 */
+	ThisTimeLineID = save_currtli;
+
+	if (record == NULL)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not read foreign transaction state from xlog at %X/%X",
+						LSN_FORMAT_ARGS(lsn))));
+
+	if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID ||
+		(XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("expected foreign transaction state data is not present in xlog at %X/%X",
+						LSN_FORMAT_ARGS(lsn))));
+
+	if (len != NULL)
+		*len = XLogRecGetDataLen(xlogreader);
+
+	*buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader));
+	memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader));
+
+	XLogReaderFree(xlogreader);
+}
+
+/*
+ * Given a transaction id, userid and serverid read it either from disk
+ * or read it directly via shmem xlog record pointer using the provided
+ * "insert_start_lsn".
+ */
+static char *
+ProcessFdwXactBuffer(TransactionId xid, Oid umid, XLogRecPtr insert_start_lsn,
+					 bool fromdisk)
+{
+	TransactionId origNextXid =
+	XidFromFullTransactionId(ShmemVariableCache->nextXid);
+	char	   *buf;
+
+	Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE));
+
+	if (!fromdisk)
+		Assert(!XLogRecPtrIsInvalid(insert_start_lsn));
+
+	/* Reject XID if too new */
+	if (TransactionIdFollowsOrEquals(xid, origNextXid))
+	{
+		if (fromdisk)
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state file for xid %u and user mapping %u",
+							xid, umid)));
+			RemoveFdwXactStateFile(xid, umid, true);
+		}
+		else
+		{
+			ereport(WARNING,
+					(errmsg("removing future fdwxact state from memory for xid %u and user mapping %u",
+							xid, umid)));
+			FdwXactRedoRemove(xid, umid, true);
+		}
+		return NULL;
+	}
+
+	if (fromdisk)
+	{
+		/* Read and validate file */
+		buf = ReadFdwXactStateFile(xid, umid);
+	}
+	else
+	{
+		/* Read xlog data */
+		XlogReadFdwXactData(insert_start_lsn, &buf, NULL);
+	}
+
+	return buf;
+}
+
+/*
+ * Read and validate the foreign transaction state file.
+ *
+ * If it looks OK (has a valid magic number and CRC), return the palloc'd
+ * contents of the file, issuing an error when finding corrupted data.
+ * This state can be reached when doing recovery.
+ */
+static char *
+ReadFdwXactStateFile(TransactionId xid, Oid umid)
+{
+	char		path[MAXPGPATH];
+	int			fd;
+	FdwXactStateOnDiskData *fdwxact_file_data;
+	struct stat stat;
+	uint32		crc_offset;
+	pg_crc32c	calc_crc;
+	pg_crc32c	file_crc;
+	char	   *buf;
+	int			r;
+
+	FdwXactStateFilePath(path, xid, umid);
+
+	fd = OpenTransientFile(path, O_RDONLY | PG_BINARY);
+	if (fd < 0)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not open FDW transaction state file \"%s\": %m",
+						path)));
+
+	/*
+	 * Check file length.  We can determine a lower bound pretty easily. We
+	 * set an upper bound to avoid palloc() failure on a corrupt file, though
+	 * we can't guarantee that we won't get an out of memory error anyway,
+	 * even on a valid file.
+	 */
+	if (fstat(fd, &stat))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not stat FDW transaction state file \"%s\": %m",
+						path)));
+
+	if (stat.st_size < (offsetof(FdwXactStateOnDiskData, identifier) +
+						sizeof(pg_crc32c)) ||
+		stat.st_size > MaxAllocSize)
+
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("too large FDW transaction state file \"%s\": %m",
+						path)));
+
+	crc_offset = stat.st_size - sizeof(pg_crc32c);
+	if (crc_offset != MAXALIGN(crc_offset))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("incorrect alignment of CRC offset for file \"%s\"",
+						path)));
+
+	/*
+	 * Ok, slurp in the file.
+	 */
+	buf = (char *) palloc(stat.st_size);
+	fdwxact_file_data = (FdwXactStateOnDiskData *) buf;
+
+	/* Slurp the file */
+	pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ);
+	r = read(fd, buf, stat.st_size);
+	if (r != stat.st_size)
+	{
+		if (r < 0)
+			ereport(ERROR,
+					(errcode_for_file_access(),
+					 errmsg("could not read file \"%s\": %m", path)));
+		else
+			ereport(ERROR,
+					(errmsg("could not read file \"%s\": read %d of %zu",
+							path, r, (Size) stat.st_size)));
+	}
+	pgstat_report_wait_end();
+
+	if (CloseTransientFile(fd))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not close file \"%s\": %m", path)));
+
+	/*
+	 * Check the CRC.
+	 */
+	INIT_CRC32C(calc_crc);
+	COMP_CRC32C(calc_crc, buf, crc_offset);
+	FIN_CRC32C(calc_crc);
+
+	file_crc = *((pg_crc32c *) (buf + crc_offset));
+
+	if (!EQ_CRC32C(calc_crc, file_crc))
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("calculated CRC checksum does not match value stored in file \"%s\"",
+						path)));
+
+	/* Check if the contents is an expected data */
+	fdwxact_file_data = (FdwXactStateOnDiskData *) buf;
+	if (fdwxact_file_data->xid != xid ||
+		fdwxact_file_data->umid != umid)
+		ereport(ERROR,
+				(errcode(ERRCODE_DATA_CORRUPTED),
+				 errmsg("invalid foreign transaction state file \"%s\"",
+						path)));
+
+	return buf;
+}
+
+/*
+ * Remove the foreign transaction file for given entry.
+ *
+ * If giveWarning is false, do not complain about file-not-present;
+ * this is an expected case during WAL replay.
+ */
+static void
+RemoveFdwXactStateFile(TransactionId xid, Oid umid, bool giveWarning)
+{
+	char		path[MAXPGPATH];
+
+	FdwXactStateFilePath(path, xid, umid);
+	if (unlink(path) < 0 && (errno != ENOENT || giveWarning))
+		ereport(WARNING,
+				(errcode_for_file_access(),
+				 errmsg("could not remove foreign transaction state file \"%s\": %m",
+						path)));
+}
+
+/*
+ * Return the list of FdwXactState entries that match at least one given criteria
+ * that is not invalid.  The caller must hold FdwXactLock.
+ */
+static List *
+find_fdwxacts(TransactionId xid, Oid umid, Oid dbid)
+{
+	List	   *res = NIL;
+
+	Assert(LWLockHeldByMe(FdwXactLock));
+
+	for (int i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		FdwXactState fdwxact = FdwXactCtl->xacts[i];
+		bool		match = false;
+
+		if (!fdwxact->valid)
+			continue;
+
+		/* xid */
+		if (TransactionIdIsValid(xid) && xid == fdwxact->data.xid)
+			match = true;
+
+		/* umid */
+		if (OidIsValid(umid) && umid == fdwxact->data.umid)
+			match = true;
+
+		/* dbid */
+		if (OidIsValid(dbid) && dbid == fdwxact->data.dbid)
+			match = true;
+
+		if (match)
+			res = lappend(res, fdwxact);
+	}
+
+	return res;
+}
+
+/*
+ * Get FdwXact entry and do some sanity checks. If check_twophase_xact is true, we
+ * also check if the given xid is prepared.  The caller must hold FdwXactLock.
+ */
+static FdwXactState
+get_fdwxact_with_check(TransactionId xid, Oid umid, bool check_twophase_xact)
+{
+	FdwXactState fdwxact = NULL;
+	Oid			myuserid;
+
+	Assert(LWLockHeldByMe(FdwXactLock));
+
+	/* Look for FdwXactState entry that matches the given xid and umid */
+	for (int i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		FdwXactState fx = FdwXactCtl->xacts[i];
+
+		if (fx->valid && fx->data.xid == xid && fx->data.umid == umid)
+		{
+			fdwxact = fx;
+			break;
+		}
+	}
+
+	/* not found */
+	if (!fdwxact)
+		return NULL;
+
+	/* check if belonging to another database */
+	if (fdwxact->data.dbid != MyDatabaseId)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("foreign transaction belongs to another database"),
+				 errhint("Connect to the database where the transaction was created to finish it.")));
+
+	/* permission check */
+	myuserid = GetUserId();
+	if (myuserid != fdwxact->data.owner && !superuser_arg(myuserid))
+		ereport(ERROR,
+				(errmsg("permission denied to resolve prepared foreign transaction"),
+				 errhint("Must be superuser or the user that prepared the transaction")));
+
+	/* check if the entry is being processed by someone */
+	if (fdwxact->locking_backend != InvalidBackendId)
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("foreign transaction with transaction identifier \"%s\" is busy",
+						fdwxact->data.identifier)));
+
+	if (check_twophase_xact && TwoPhaseExists(fdwxact->data.xid))
+	{
+		/*
+		 * the entry's local transaction is prepared. Since we cannot know the
+		 * fate of the local transaction, we cannot resolve this foreign
+		 * transaction.
+		 */
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("cannot resolve foreign transaction with identifier \"%s\" whose local transaction is in-progress",
+						fdwxact->data.identifier),
+				 errhint("Do COMMIT PREPARED or ROLLBACK PREPARED")));
+	}
+
+	return fdwxact;
+}
+
+/* Error cleanup callback for pg_foreign_resolve/remove_xact */
+static void
+pg_foreign_xact_callback(int code, Datum arg)
+{
+	FdwXactState fdwxact = (FdwXactState) DatumGetPointer(arg);
+
+	if (fdwxact->valid)
+	{
+		Assert(fdwxact->locking_backend == MyBackendId);
+
+		LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+		fdwxact->locking_backend = InvalidBackendId;
+		LWLockRelease(FdwXactLock);
+	}
+}
+
+/* Built in functions */
+
+/*
+ * Structure to hold and iterate over the foreign transactions to be displayed
+ * by the built-in functions.
+ */
+typedef struct
+{
+	FdwXactState fdwxacts;
+	int			num_xacts;
+	int			cur_xact;
+}			WorkingStatus;
+
+Datum
+pg_foreign_xacts(PG_FUNCTION_ARGS)
+{
+#define PG_PREPARED_FDWXACTS_COLS	7
+	ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
+	TupleDesc	tupdesc;
+	Tuplestorestate *tupstore;
+	MemoryContext per_query_ctx;
+	MemoryContext oldcontext;
+
+	/* check to see if caller supports us returning a tuplestore */
+	if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("set-valued function called in context that cannot accept a set")));
+	if (!(rsinfo->allowedModes & SFRM_Materialize))
+		ereport(ERROR,
+				(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+				 errmsg("materialize mode required, but it is not allowed in this context")));
+
+	/* Build a tuple descriptor for our result type */
+	if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
+		elog(ERROR, "return type must be a row type");
+
+	per_query_ctx = rsinfo->econtext->ecxt_per_query_memory;
+	oldcontext = MemoryContextSwitchTo(per_query_ctx);
+
+	tupstore = tuplestore_begin_heap(true, false, work_mem);
+	rsinfo->returnMode = SFRM_Materialize;
+	rsinfo->setResult = tupstore;
+	rsinfo->setDesc = tupdesc;
+
+	MemoryContextSwitchTo(oldcontext);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		FdwXactState fdwxact = FdwXactCtl->xacts[i];
+		FdwXactStatus status;
+		char	   *xact_status;
+		Datum		values[PG_PREPARED_FDWXACTS_COLS];
+		bool		nulls[PG_PREPARED_FDWXACTS_COLS];
+
+		if (!fdwxact->valid)
+			continue;
+
+		memset(nulls, 0, sizeof(nulls));
+
+		SpinLockAcquire(&fdwxact->mutex);
+		status = fdwxact->status;
+		SpinLockRelease(&fdwxact->mutex);
+
+		values[0] = TransactionIdGetDatum(fdwxact->data.xid);
+		values[1] = ObjectIdGetDatum(fdwxact->data.umid);
+		values[2] = ObjectIdGetDatum(fdwxact->data.owner);
+		values[3] = ObjectIdGetDatum(fdwxact->data.dbid);
+
+		switch (status)
+		{
+			case FDWXACT_STATUS_PREPARING:
+				xact_status = "preparing";
+				break;
+			case FDWXACT_STATUS_PREPARED:
+				xact_status = "prepared";
+				break;
+			case FDWXACT_STATUS_COMMITTING:
+				xact_status = "committing";
+				break;
+			case FDWXACT_STATUS_ABORTING:
+				xact_status = "aborting";
+				break;
+			default:
+				xact_status = "unknown";
+				break;
+		}
+
+		values[4] = CStringGetTextDatum(xact_status);
+		values[5] = CStringGetTextDatum(fdwxact->data.identifier);
+
+		if (fdwxact->locking_backend != InvalidBackendId)
+		{
+			PGPROC	   *locker = BackendIdGetProc(fdwxact->locking_backend);
+
+			values[6] = Int32GetDatum(locker->pid);
+		}
+		else
+			nulls[6] = true;
+
+		tuplestore_putvalues(tupstore, tupdesc, values, nulls);
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* clean up and return the tuplestore */
+	tuplestore_donestoring(tupstore);
+
+	return (Datum) 0;
+}
+
+/*
+ * Built-in SQL function to resolve a prepared foreign transaction.
+ */
+Datum
+pg_resolve_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			umid = PG_GETARG_OID(1);
+	FdwXactState fdwxact;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_fdwxact_with_check(xid, umid, true);
+
+	/* lock it */
+	fdwxact->locking_backend = MyBackendId;
+
+	LWLockRelease(FdwXactLock);
+
+	/*
+	 * Resolve the foreign transaction.  We ensure unlocking FdwXact entry at
+	 * an error or an interruption.
+	 *
+	 * XXX we assume that an interruption doesn't happen between locking
+	 * FdwXact entry and registering the callback, especially in
+	 * LWLockRelease().
+	 */
+	PG_ENSURE_ERROR_CLEANUP(pg_foreign_xact_callback,
+							(Datum) PointerGetDatum(fdwxact));
+	{
+		ResolveOneFdwXact(fdwxact);
+	}
+	PG_END_ENSURE_ERROR_CLEANUP(pg_foreign_xact_callback,
+								(Datum) PointerGetDatum(fdwxact));
+
+	PG_RETURN_BOOL(true);
+}
+
+/*
+ * Built-in function to remove a prepared foreign transaction entry without
+ * resolution. The function gives a way to forget about such prepared
+ * transaction in case: the foreign server where it is prepared is no longer
+ * available, the user which prepared this transaction needs to be dropped.
+ */
+Datum
+pg_remove_foreign_xact(PG_FUNCTION_ARGS)
+{
+	TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0));
+	Oid			umid = PG_GETARG_OID(1);
+	FdwXactState fdwxact;
+
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+
+	fdwxact = get_fdwxact_with_check(xid, umid, false);
+
+	/* Clean up entry and any files we may have left */
+	remove_fdwxact(fdwxact);
+
+	LWLockRelease(FdwXactLock);
+
+	PG_RETURN_BOOL(true);
 }
diff --git a/src/backend/access/transam/fdwxact_launcher.c b/src/backend/access/transam/fdwxact_launcher.c
new file mode 100644
index 0000000000..59f061773c
--- /dev/null
+++ b/src/backend/access/transam/fdwxact_launcher.c
@@ -0,0 +1,587 @@
+/*-------------------------------------------------------------------------
+ *
+ * launcher.c
+ *
+ * The foreign transaction resolver launcher process starts foreign
+ * transaction resolver processes. The launcher schedules resolver
+ * process to be started when requested by backend process.
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/transam/fdwxact_launcher.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "funcapi.h"
+#include "pgstat.h"
+#include "funcapi.h"
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
+#include "access/resolver_internal.h"
+#include "access/twophase.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "postmaster/bgworker.h"
+#include "postmaster/interrupt.h"
+#include "storage/ipc.h"
+#include "storage/proc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+FdwXactResolver *MyFdwXactResolver = NULL;
+
+static volatile sig_atomic_t got_SIGUSR2 = false;
+
+static void FdwXactLauncherOnExit(int code, Datum arg);
+static void LaunchFdwXactResolver(Oid dbid, Oid serverid);
+static bool RelaunchFdwXactResolvers(void);
+static FdwXactResolver *FdwXactResolverFind(Oid dbid, Oid serverid);
+
+/* Signal handler */
+static void FdwXactLaunchHandler(SIGNAL_ARGS);
+
+
+/*
+ * Wake up the launcher process to request launching new resolvers
+ * immediately.
+ */
+void
+RequestToLaunchFdwXactResolver(void)
+{
+	if (FdwXactResolverCtl->launcher_pid != InvalidPid)
+		kill(FdwXactResolverCtl->launcher_pid, SIGUSR2);
+}
+
+/* Report shared memory space needed by FdwXactLauncherShmemInit */
+Size
+FdwXactLauncherShmemSize(void)
+{
+	Size		size = 0;
+
+	size = add_size(size, SizeOfFdwXactResolverCtlData);
+	size = add_size(size, mul_size(max_foreign_xact_resolvers,
+								   sizeof(FdwXactResolver)));
+
+	return size;
+}
+
+/*
+ * Allocate and initialize foreign transaction resolver shared
+ * memory.
+ */
+void
+FdwXactLauncherShmemInit(void)
+{
+	bool		found;
+
+	FdwXactResolverCtl = ShmemInitStruct("Foreign Transaction Launcher Data",
+										 FdwXactLauncherShmemSize(),
+										 &found);
+
+	if (!IsUnderPostmaster)
+	{
+		int			slot;
+
+		/* First time through, so initialize */
+		MemSet(FdwXactResolverCtl, 0, FdwXactLauncherShmemSize());
+		FdwXactResolverCtl->launcher_pid = InvalidPid;
+
+		for (slot = 0; slot < max_foreign_xact_resolvers; slot++)
+		{
+			FdwXactResolver *resolver = &FdwXactResolverCtl->resolvers[slot];
+
+			memset(resolver, 0, sizeof(FdwXactResolver));
+			SpinLockInit(&(resolver->mutex));
+		}
+	}
+}
+
+/*
+ * Cleanup function for fdwxact launcher
+ *
+ * Called on fdwxact launcher exit.
+ */
+static void
+FdwXactLauncherOnExit(int code, Datum arg)
+{
+	FdwXactResolverCtl->launcher_pid = InvalidPid;
+}
+
+/* SIGUSR2: set flag to launch new resolver process immediately */
+static void
+FdwXactLaunchHandler(SIGNAL_ARGS)
+{
+	int			save_errno = errno;
+
+	got_SIGUSR2 = true;
+	SetLatch(MyLatch);
+
+	errno = save_errno;
+}
+
+/*
+ * Main loop for the fdwxact launcher process.
+ */
+void
+FdwXactLauncherMain(Datum main_arg)
+{
+	TimestampTz last_start_time = 0;
+
+	ereport(DEBUG1,
+			(errmsg("fdwxact resolver launcher started")));
+
+	before_shmem_exit(FdwXactLauncherOnExit, (Datum) 0);
+
+	Assert(FdwXactResolverCtl->launcher_pid == InvalidPid);
+	FdwXactResolverCtl->launcher_pid = MyProcPid;
+	FdwXactResolverCtl->launcher_latch = &MyProc->procLatch;
+
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGUSR2, FdwXactLaunchHandler);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	BackgroundWorkerInitializeConnection(NULL, NULL, 0);
+
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz now;
+		long		wait_time = DEFAULT_NAPTIME_PER_CYCLE;
+		int			rc;
+
+		CHECK_FOR_INTERRUPTS();
+		ResetLatch(MyLatch);
+
+		now = GetCurrentTimestamp();
+
+		/*
+		 * Limit the start retry to once a
+		 * foreign_xact_resolution_retry_interval but always attempt to start
+		 * when requested.
+		 */
+		if (got_SIGUSR2 ||
+			TimestampDifferenceExceeds(last_start_time, now,
+									   foreign_xact_resolution_retry_interval))
+		{
+			MemoryContext oldctx;
+			MemoryContext subctx;
+			bool		launched;
+
+			if (got_SIGUSR2)
+				got_SIGUSR2 = false;
+
+			subctx = AllocSetContextCreate(TopMemoryContext,
+										   "Foreign Transaction Launcher",
+										   ALLOCSET_DEFAULT_SIZES);
+			oldctx = MemoryContextSwitchTo(subctx);
+
+			launched = RelaunchFdwXactResolvers();
+			if (launched)
+			{
+				last_start_time = now;
+				wait_time = foreign_xact_resolution_retry_interval;
+			}
+
+			/* Switch back to original memory context. */
+			MemoryContextSwitchTo(oldctx);
+
+			/* Clean the temporary memory. */
+			MemoryContextDelete(subctx);
+		}
+		else
+		{
+			/*
+			 * The wait in previous cycle was interrupted in less than
+			 * foreign_xact_resolution_retry_interval since last resolver
+			 * started, this usually means crash of the resolver, so we should
+			 * retry in foreign_xact_resolution_retry_interval again.
+			 */
+			wait_time = foreign_xact_resolution_retry_interval;
+		}
+
+		/* Wait for more work */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   wait_time,
+					   WAIT_EVENT_FDWXACT_LAUNCHER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		if (ConfigReloadPending)
+		{
+			ConfigReloadPending = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+	}
+
+	/* Not reachable */
+}
+
+/*
+ * Request the launcher to launch or wakeup foreign transaction resolvers for
+ * the given servers.
+ */
+void
+LaunchOrWakeupFdwXactResolver(List *serveroids_orig)
+{
+	List	   *wakeup_resolvers = NIL;
+	List	   *serveroids = list_copy(serveroids_orig);
+
+	/* Collect running resolvers to wakeup */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (int i = 0;
+		 i < max_foreign_xact_resolvers && list_length(serveroids) > 0;
+		 i++)
+	{
+		FdwXactResolver *resolver = &FdwXactResolverCtl->resolvers[i];
+
+		if (resolver->in_use &&
+			resolver->dbid == MyDatabaseId &&
+			list_member_oid(serveroids, resolver->serverid))
+		{
+			wakeup_resolvers = lappend(wakeup_resolvers, resolver);
+			serveroids = list_delete_oid(serveroids, resolver->serverid);
+		}
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	if (wakeup_resolvers != NIL)
+	{
+		ListCell   *lc;
+
+		foreach(lc, wakeup_resolvers)
+		{
+			FdwXactResolver *resolver = (FdwXactResolver *) lfirst(lc);
+
+			/*
+			 * Wakeup the resolver. It's possible that the resolver is
+			 * starting up and doesn't attach its slot yet. Since the resolver
+			 * will find FdwXactState entry we inserted soon we don't
+			 * anything.
+			 */
+			if (resolver->latch)
+				SetLatch(resolver->latch);
+		}
+
+		list_free(wakeup_resolvers);
+	}
+
+	/* Request to launch new resolvers if any */
+	if (list_length(serveroids) > 0)
+		RequestToLaunchFdwXactResolver();
+}
+
+/*
+ * Launch a foreign transaction resolver process that will connect to given
+ * 'dbid' and 'serverid'.
+ */
+static void
+LaunchFdwXactResolver(Oid dbid, Oid serverid)
+{
+	BackgroundWorker bgw;
+	BackgroundWorkerHandle *bgw_handle;
+	FdwXactResolver *resolver;
+	int			unused_slot = -1;
+	int			i;
+
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	/* Find unused resolver slot */
+	for (i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactResolverCtl->resolvers[i];
+
+		if (!resolver->in_use)
+		{
+			unused_slot = i;
+			break;
+		}
+	}
+
+	/* No unused found */
+	if (i >= max_foreign_xact_resolvers)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of foreign transaction resolver slots"),
+				 errhint("You might need to increase max_foreign_transaction_resolvers.")));
+
+	resolver = &FdwXactResolverCtl->resolvers[unused_slot];
+	resolver->in_use = true;
+	resolver->dbid = dbid;
+	resolver->serverid = serverid;
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Register the new dynamic worker */
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction resolver for server %u on database %u",
+			 resolver->serverid, resolver->dbid);
+	snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver");
+	bgw.bgw_restart_time = BGW_NEVER_RESTART;
+	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_main_arg = Int32GetDatum(unused_slot);
+
+	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
+	{
+		/* Failed to launch, cleanup the worker slot */
+		SpinLockAcquire(&(MyFdwXactResolver->mutex));
+		resolver->in_use = false;
+		SpinLockRelease(&(MyFdwXactResolver->mutex));
+
+		ereport(WARNING,
+				(errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED),
+				 errmsg("out of background worker slots"),
+				 errhint("You might need to increase max_worker_processes.")));
+	}
+
+	/*
+	 * We don't need to wait until it attaches here because we're going to
+	 * wait until all foreign transactions are resolved.
+	 */
+}
+
+/*
+ * Check the pending foreign transaction entries and (re)launch the resolvers
+ * for them if necessary.  Return true if launched at least one resolver.
+ */
+static bool
+RelaunchFdwXactResolvers(void)
+{
+	HTAB	   *launch_resolvers;
+	HASHCTL		ctl;
+	HASH_SEQ_STATUS status;
+	struct fdwxact_resolver_entry
+	{
+		Oid			dbid;
+		Oid			serverid;
+	};
+	struct fdwxact_resolver_entry *entry;
+
+	/* Create a hash map for resolvers that need to launch */
+	memset(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(struct fdwxact_resolver_entry);
+	ctl.entrysize = sizeof(struct fdwxact_resolver_entry);
+	launch_resolvers = hash_create("fdwxact resolvers to launch",
+								   32, &ctl, HASH_ELEM | HASH_BLOBS);
+
+	LWLockAcquire(FdwXactLock, LW_SHARED);
+	for (int i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		FdwXactState fdwxact = FdwXactCtl->xacts[i];
+
+		if (!fdwxact->valid)
+			continue;
+
+		/*
+		 * We need to launch resolver process if the foreign transaction is
+		 * not held by anyone and is not a part of the local prepared
+		 * transaction.
+		 */
+		if (fdwxact->locking_backend == InvalidBackendId &&
+			!TwoPhaseExists(fdwxact->data.xid))
+		{
+			struct fdwxact_resolver_entry ent = {
+				.dbid = fdwxact->data.dbid,
+				.serverid = fdwxact->data.serverid
+			};
+
+			hash_search(launch_resolvers, (void *) &ent, HASH_ENTER, NULL);
+		}
+	}
+	LWLockRelease(FdwXactLock);
+
+	/* There is no foreign transaction to resolve, no need to launch new one */
+	if (hash_get_num_entries(launch_resolvers) == 0)
+	{
+		hash_destroy(launch_resolvers);
+		return false;
+	}
+
+	/* Check if the resolver is already running */
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	for (int i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactResolverCtl->resolvers[i];
+		struct fdwxact_resolver_entry ent;
+
+		if (!resolver->in_use)
+			continue;
+
+		ent.dbid = resolver->dbid;
+		ent.serverid = resolver->serverid;
+		hash_search(launch_resolvers, &ent, HASH_REMOVE, NULL);
+	}
+	LWLockRelease(FdwXactResolverLock);
+
+	/* Return if resolvers for all foreign transactions are already running */
+	if (hash_get_num_entries(launch_resolvers) == 0)
+	{
+		hash_destroy(launch_resolvers);
+		return false;
+	}
+
+	/* Launch new resolvers */
+	hash_seq_init(&status, launch_resolvers);
+	while ((entry = (struct fdwxact_resolver_entry *) hash_seq_search(&status)) != NULL)
+		LaunchFdwXactResolver(entry->dbid, entry->serverid);
+
+	hash_destroy(launch_resolvers);
+	return true;
+}
+
+/* Register a background worker running the foreign transaction launcher */
+void
+FdwXactLauncherRegister(void)
+{
+	BackgroundWorker bgw;
+
+	if (max_foreign_xact_resolvers == 0)
+		return;
+
+	memset(&bgw, 0, sizeof(bgw));
+	bgw.bgw_flags = BGWORKER_SHMEM_ACCESS |
+		BGWORKER_BACKEND_DATABASE_CONNECTION;
+	bgw.bgw_start_time = BgWorkerStart_RecoveryFinished;
+	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
+	snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain");
+	snprintf(bgw.bgw_name, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	snprintf(bgw.bgw_type, BGW_MAXLEN,
+			 "foreign transaction launcher");
+	bgw.bgw_restart_time = 5;
+	bgw.bgw_notify_pid = 0;
+	bgw.bgw_main_arg = (Datum) 0;
+
+	RegisterBackgroundWorker(&bgw);
+}
+
+bool
+IsFdwXactLauncher(void)
+{
+	return FdwXactResolverCtl->launcher_pid == MyProcPid;
+}
+
+/*
+ * Return the foreign transaction resolver running for dbid and serverid
+ */
+FdwXactResolver *
+FdwXactResolverFind(Oid dbid, Oid serverid)
+{
+	Assert(LWLockHeldByMe(FdwXactResolverLock));
+	for (int i = 0; i < max_foreign_xact_resolvers; i++)
+	{
+		FdwXactResolver *resolver = &FdwXactResolverCtl->resolvers[i];
+
+		if (resolver->in_use && resolver->dbid == dbid &&
+			resolver->serverid == serverid)
+			return resolver;
+	}
+
+	return NULL;
+}
+
+/*
+ * Stop the given foreign transaction resolver, and wait until it detaches
+ * from slot.
+ */
+void
+StopFdwXactResolver(Oid dbid, Oid serverid)
+{
+	FdwXactResolver *resolver;
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+
+	resolver = FdwXactResolverFind(dbid, serverid);
+
+	/* Not worker, nothing to do */
+	if (!resolver)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		return;
+	}
+
+	/* Found the resolver, terminate it ... */
+	kill(resolver->pid, SIGTERM);
+
+	/* ... and wait for it to die */
+	for (;;)
+	{
+		int			rc;
+
+		/* is it gone? */
+		if (!resolver->in_use)
+			break;
+
+		LWLockRelease(FdwXactResolverLock);
+
+		/* Wait a bit --- we don't expect to have to wait long. */
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+					   10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
+
+		if (rc & WL_LATCH_SET)
+		{
+			ResetLatch(MyLatch);
+			CHECK_FOR_INTERRUPTS();
+		}
+
+		LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	}
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/*
+ * Stop the fdwxact resolver running for the given foreign server.
+ */
+Datum
+pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS)
+{
+	char	   *servername = text_to_cstring(PG_GETARG_TEXT_P(0));
+	ForeignServer *server;
+	FdwXactResolver *resolver;
+
+	/* Must be super user */
+	if (!superuser())
+		ereport(ERROR,
+				(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
+				 errmsg("permission denied to stop foreign transaction resolver")));
+
+	server = GetForeignServerByName(servername, false);
+
+	LWLockAcquire(FdwXactResolverLock, LW_SHARED);
+	resolver = FdwXactResolverFind(MyDatabaseId, server->serverid);
+	LWLockRelease(FdwXactResolverLock);
+
+	if (!resolver)
+		ereport(ERROR,
+				(errmsg("there is no running foreign transaction resolver process on server %s",
+						servername)));
+
+	StopFdwXactResolver(MyDatabaseId, server->serverid);
+	pfree(server);
+
+	PG_RETURN_BOOL(true);
+}
diff --git a/src/backend/access/transam/fdwxact_resolver.c b/src/backend/access/transam/fdwxact_resolver.c
new file mode 100644
index 0000000000..587c4759ca
--- /dev/null
+++ b/src/backend/access/transam/fdwxact_resolver.c
@@ -0,0 +1,339 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver.c
+ *
+ * The foreign transaction resolver background worker resolves in-doubt
+ * foreign transactions, foreign transactions participate to a distributed
+ * transaction but aren't being processed anyone.  A resolver process is
+ * launched per foreign server by foreign transaction launcher.
+ *
+ * Normal termination is by SIGTERM, which instructs the resolver process
+ * to exit(0) at the next convenient moment. Emergency termination is by
+ * SIGQUIT; like any backend. The resolver process also terminate by timeouts
+ * only if there is no pending foreign transactions on the database waiting
+ * to be resolved.
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/transam/fdwxact_resolver.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include <signal.h>
+#include <unistd.h>
+
+#include "access/fdwxact.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
+#include "access/resolver_internal.h"
+#include "access/transam.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "commands/dbcommands.h"
+#include "funcapi.h"
+#include "libpq/libpq.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "postmaster/bgworker.h"
+#include "postmaster/interrupt.h"
+#include "storage/ipc.h"
+#include "tcop/tcopprot.h"
+#include "utils/builtins.h"
+#include "utils/timeout.h"
+#include "utils/timestamp.h"
+
+/* max sleep time between cycles (3min) */
+#define DEFAULT_NAPTIME_PER_CYCLE 180000L
+
+/* GUC parameters */
+int			foreign_xact_resolution_retry_interval;
+int			foreign_xact_resolver_timeout = 60 * 1000;
+
+FdwXactResolverCtlData *FdwXactResolverCtl;
+
+static void FdwXactResolverLoop(void);
+static long FdwXactResolverComputeSleepTime(TimestampTz now);
+static void FdwXactResolverCheckTimeout(TimestampTz now);
+
+static void FdwXactResolverOnExit(int code, Datum arg);
+static void FdwXactResolverDetach(void);
+static void FdwXactResolverAttach(int slot);
+static void FdwXactResolverProcessInDoubtXacts(void);
+
+/* The list of currently holding FdwXact entries. */
+static List *heldFdwXactEntries = NIL;
+
+static TimestampTz last_resolution_time = -1;
+
+/*
+ * Detach the resolver and cleanup the resolver info.
+ */
+static void
+FdwXactResolverDetach(void)
+{
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	MyFdwXactResolver->pid = InvalidPid;
+	MyFdwXactResolver->in_use = false;
+	MyFdwXactResolver->dbid = InvalidOid;
+	MyFdwXactResolver->serverid = InvalidOid;
+
+	LWLockRelease(FdwXactResolverLock);
+
+	/*
+	 * Force to send remaining WAL statistics to the stats collector at
+	 * process exit.
+	 *
+	 * Since pgstat_send_wal is invoked with 'force' is false in main loop
+	 * to avoid overloading to the stats collector, there may exist unsent
+	 * stats counters for the WAL writer.
+	 */
+	pgstat_send_wal(true);
+}
+
+/*
+ * Cleanup up foreign transaction resolver info and releas the holding
+ * FdwXactState entries.
+ */
+static void
+FdwXactResolverOnExit(int code, Datum arg)
+{
+	ListCell   *lc;
+
+	FdwXactResolverDetach();
+
+	/* Release the held foreign transaction entries */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	foreach(lc, heldFdwXactEntries)
+	{
+		FdwXactState fdwxact = (FdwXactState) lfirst(lc);
+
+		if (fdwxact->valid && fdwxact->locking_backend == MyBackendId)
+			fdwxact->locking_backend = InvalidBackendId;
+	}
+	LWLockRelease(FdwXactLock);
+}
+
+/*
+ * Attach to a slot.
+ */
+static void
+FdwXactResolverAttach(int slot)
+{
+	/* Block concurrent access */
+	LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE);
+
+	Assert(slot >= 0 && slot < max_foreign_xact_resolvers);
+	MyFdwXactResolver = &FdwXactResolverCtl->resolvers[slot];
+
+	if (!MyFdwXactResolver->in_use)
+	{
+		LWLockRelease(FdwXactResolverLock);
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("foreign transaction resolver slot %d is empty, cannot attach",
+						slot)));
+	}
+
+	Assert(OidIsValid(MyFdwXactResolver->dbid));
+	Assert(OidIsValid(MyFdwXactResolver->serverid));
+
+	MyFdwXactResolver->pid = MyProcPid;
+	MyFdwXactResolver->latch = &MyProc->procLatch;
+
+	before_shmem_exit(FdwXactResolverOnExit, (Datum) 0);
+
+	LWLockRelease(FdwXactResolverLock);
+}
+
+/* Foreign transaction resolver entry point */
+void
+FdwXactResolverMain(Datum main_arg)
+{
+	int			slot = DatumGetInt32(main_arg);
+	char	   *datname;
+	ForeignServer *server;
+
+	/* Attach to a slot */
+	FdwXactResolverAttach(slot);
+
+	/* Establish signal handlers */
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
+
+	/* Connect to our database */
+	BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0);
+
+	StartTransactionCommand();
+	datname = get_database_name(MyFdwXactResolver->dbid);
+	server = GetForeignServer(MyFdwXactResolver->serverid);
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for server \"%s\" on database \"%s\" has started",
+					server->servername, datname)));
+	pfree(datname);
+	pfree(server);
+	CommitTransactionCommand();
+
+	/* Initialize stats to a sanish value */
+	last_resolution_time = GetCurrentTimestamp();
+
+	/* Run the main loop */
+	FdwXactResolverLoop();
+
+	proc_exit(0);
+}
+
+/*
+ * Fdwxact resolver main loop
+ */
+static void
+FdwXactResolverLoop(void)
+{
+	/* Enter main loop */
+	for (;;)
+	{
+		TimestampTz now;
+		int			rc;
+		long		sleep_time = DEFAULT_NAPTIME_PER_CYCLE;
+
+		ResetLatch(MyLatch);
+
+		CHECK_FOR_INTERRUPTS();
+
+		if (ConfigReloadPending)
+		{
+			ConfigReloadPending = false;
+			ProcessConfigFile(PGC_SIGHUP);
+		}
+
+		/* Resolve in-doubt transactions if any  */
+		FdwXactResolverProcessInDoubtXacts();
+
+		now = GetCurrentTimestamp();
+		FdwXactResolverCheckTimeout(now);
+		sleep_time = FdwXactResolverComputeSleepTime(now);
+
+		/* Send WAL statistics to the stats collector */
+		pgstat_send_wal(false);
+
+		rc = WaitLatch(MyLatch,
+					   WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH,
+					   sleep_time,
+					   WAIT_EVENT_FDWXACT_RESOLVER_MAIN);
+
+		if (rc & WL_POSTMASTER_DEATH)
+			proc_exit(1);
+	}
+}
+
+/*
+ * Check whether there have been foreign transactions by the backend within
+ * foreign_xact_resolver_timeout and shutdown if not.
+ */
+static void
+FdwXactResolverCheckTimeout(TimestampTz now)
+{
+	TimestampTz timeout;
+	ForeignServer *server;
+
+	if (foreign_xact_resolver_timeout == 0)
+		return;
+
+	timeout = TimestampTzPlusMilliseconds(last_resolution_time,
+										  foreign_xact_resolver_timeout);
+
+	if (now < timeout)
+		return;
+
+	/* Reached timeout, exit */
+	StartTransactionCommand();
+	server = GetForeignServer(MyFdwXactResolver->serverid);
+	ereport(LOG,
+			(errmsg("foreign transaction resolver for server \"%s\" on database \"%s\" will stop because the timeout",
+					server->servername,
+					get_database_name(MyDatabaseId))));
+	CommitTransactionCommand();
+	FdwXactResolverDetach();
+	proc_exit(0);
+}
+
+/*
+ * Compute how long we should sleep by the next cycle. We can sleep until the time
+ * out.
+ */
+static long
+FdwXactResolverComputeSleepTime(TimestampTz now)
+{
+	long		sleeptime = DEFAULT_NAPTIME_PER_CYCLE;
+
+	if (foreign_xact_resolver_timeout > 0)
+	{
+		TimestampTz timeout;
+
+		/* Compute relative time until wakeup. */
+		timeout = TimestampTzPlusMilliseconds(last_resolution_time,
+											  foreign_xact_resolver_timeout);
+		sleeptime = TimestampDifferenceMilliseconds(now, timeout);
+	}
+
+	return sleeptime;
+}
+
+bool
+IsFdwXactResolver(void)
+{
+	return MyFdwXactResolver != NULL;
+}
+
+/*
+ * Process in-doubt foreign transactions.
+ */
+static void
+FdwXactResolverProcessInDoubtXacts(void)
+{
+	ListCell   *lc;
+
+	Assert(heldFdwXactEntries == NIL);
+
+	/* Hold all in-doubt foreign transactions */
+	LWLockAcquire(FdwXactLock, LW_EXCLUSIVE);
+	for (int i = 0; i < FdwXactCtl->num_xacts; i++)
+	{
+		FdwXactState fdwxact = FdwXactCtl->xacts[i];
+
+		if (fdwxact->valid &&
+			fdwxact->locking_backend == InvalidBackendId &&
+			fdwxact->data.dbid == MyFdwXactResolver->dbid &&
+			fdwxact->data.serverid == MyFdwXactResolver->serverid &&
+			!TwoPhaseExists(fdwxact->data.xid))
+		{
+			fdwxact->locking_backend = MyBackendId;
+			heldFdwXactEntries = lappend(heldFdwXactEntries, fdwxact);
+		}
+	}
+	LWLockRelease(FdwXactLock);
+
+	foreach(lc, heldFdwXactEntries)
+	{
+		FdwXactState fdwxact = (FdwXactState) lfirst(lc);
+
+		/*
+		 * Resolve one foreign transaction. ResolveOneFdwXact() releases and
+		 * removes FdwXactState entry after resolution.
+		 */
+		StartTransactionCommand();
+		ResolveOneFdwXact(fdwxact);
+		CommitTransactionCommand();
+	}
+
+	if (list_length(heldFdwXactEntries) > 0)
+		last_resolution_time = GetCurrentTimestamp();
+
+	list_free(heldFdwXactEntries);
+	heldFdwXactEntries = NIL;
+}
diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c
index 58091f6b52..0a3f4b383f 100644
--- a/src/backend/access/transam/rmgr.c
+++ b/src/backend/access/transam/rmgr.c
@@ -10,6 +10,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index f67d813c56..29980d56ac 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -77,6 +77,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/htup_details.h"
 #include "access/subtrans.h"
 #include "access/transam.h"
@@ -846,6 +847,34 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held)
 	return result;
 }
 
+/*
+ * TwoPhaseExists
+ *		Return true if there is a prepared transaction specified by XID
+ */
+bool
+TwoPhaseExists(TransactionId xid)
+{
+	int		i;
+	bool	found = false;
+
+	LWLockAcquire(TwoPhaseStateLock, LW_SHARED);
+
+	for (i = 0; i < TwoPhaseState->numPrepXacts; i++)
+	{
+		GlobalTransaction gxact = TwoPhaseState->prepXacts[i];
+
+		if (gxact->xid == xid)
+		{
+			found = true;
+			break;
+		}
+	}
+
+	LWLockRelease(TwoPhaseStateLock);
+
+	return found;
+}
+
 /*
  * TwoPhaseGetDummyBackendId
  *		Get the dummy backend ID for prepared transaction specified by XID
@@ -1556,6 +1585,12 @@ FinishPreparedTransaction(const char *gid, bool isCommit)
 	/* Count the prepared xact as committed or aborted */
 	AtEOXact_PgStat(isCommit, false);
 
+	/*
+	 * If the prepared transaction was a part of a distributed transaction
+	 * notify a resolver process to handle it.
+	 */
+	FdwXactLaunchResolversForXid(xid);
+
 	/*
 	 * And now we can clean up any files we may have left.
 	 */
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 1e00a3a98e..d007c97c75 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -2131,7 +2131,7 @@ CommitTransaction(void)
 		AtEOXact_Parallel(true);
 
 	/* Call foreign transaction callbacks at pre-commit phase, if any */
-	AtEOXact_FdwXact(true, is_parallel_worker);
+	PreCommit_FdwXact(is_parallel_worker);
 
 	/* Shut down the deferred-trigger manager */
 	AfterTriggerEndXact(true);
@@ -2286,6 +2286,7 @@ CommitTransaction(void)
 	AtEOXact_PgStat(true, is_parallel_worker);
 	AtEOXact_Snapshot(true, false);
 	AtEOXact_ApplyLauncher(true);
+	AtEOXact_FdwXact(true, is_parallel_worker);
 	pgstat_report_xact_timestamp(0);
 
 	CurrentResourceOwner = NULL;
@@ -2559,6 +2560,7 @@ PrepareTransaction(void)
 	PostPrepare_Twophase();
 
 	/* PREPARE acts the same as COMMIT as far as GUC is concerned */
+	AtEOXact_FdwXact(true, false);
 	AtEOXact_GUC(true, 1);
 	AtEOXact_SPI(true);
 	AtEOXact_Enum();
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 9cbca6392d..837e0f198a 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -24,6 +24,7 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
@@ -4670,6 +4671,7 @@ InitControlFile(uint64 sysidentifier)
 	ControlFile->max_worker_processes = max_worker_processes;
 	ControlFile->max_wal_senders = max_wal_senders;
 	ControlFile->max_prepared_xacts = max_prepared_xacts;
+	ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 	ControlFile->max_locks_per_xact = max_locks_per_xact;
 	ControlFile->wal_level = wal_level;
 	ControlFile->wal_log_hints = wal_log_hints;
@@ -6460,6 +6462,9 @@ CheckRequiredParameterValues(void)
 		RecoveryRequiresIntParameter("max_prepared_transactions",
 									 max_prepared_xacts,
 									 ControlFile->max_prepared_xacts);
+		RecoveryRequiresIntParameter("max_prepared_foreign_transactions",
+									 max_prepared_foreign_xacts,
+									 ControlFile->max_prepared_foreign_xacts);
 		RecoveryRequiresIntParameter("max_locks_per_transaction",
 									 max_locks_per_xact,
 									 ControlFile->max_locks_per_xact);
@@ -7009,14 +7014,15 @@ StartupXLOG(void)
 	restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI);
 
 	/*
-	 * Before running in recovery, scan pg_twophase and fill in its status to
-	 * be able to work on entries generated by redo.  Doing a scan before
-	 * taking any recovery action has the merit to discard any 2PC files that
-	 * are newer than the first record to replay, saving from any conflicts at
-	 * replay.  This avoids as well any subsequent scans when doing recovery
-	 * of the on-disk two-phase data.
+	 * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then
+	 * fill in its status to be able to work on entries generated by redo.
+	 * Doing a scan before taking any recovery action has the merit to discard
+	 * any state files that are newer than the first record to replay, saving
+	 * from any conflicts at replay.  This avoids as well any subsequent scans
+	 * when doing recovery of the on-disk two-phase or fdwxact data.
 	 */
 	restoreTwoPhaseData();
+	RestoreFdwXactData();
 
 	lastFullPageWrites = checkPoint.fullPageWrites;
 
@@ -7218,7 +7224,10 @@ StartupXLOG(void)
 			InitRecoveryTransactionEnvironment();
 
 			if (wasShutdown)
+			{
 				oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+				oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
+			}
 			else
 				oldestActiveXID = checkPoint.oldestActiveXid;
 			Assert(TransactionIdIsValid(oldestActiveXID));
@@ -7730,11 +7739,13 @@ StartupXLOG(void)
 	}
 
 	/*
-	 * Pre-scan prepared transactions to find out the range of XIDs present.
-	 * This information is not quite needed yet, but it is positioned here so
-	 * as potential problems are detected before any on-disk change is done.
+	 * Pre-scan prepared transactions and foreign prepared transacftions to find
+	 * out the range of XIDs present.  This information is not quite needed yet,
+	 * but it is positioned here so as potential problems are detected before any
+	 * on-disk change is done.
 	 */
 	oldestActiveXID = PrescanPreparedTransactions(NULL, NULL);
+	oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 	/*
 	 * Allow ordinary WAL segment creation before any exitArchiveRecovery(),
@@ -8066,8 +8077,12 @@ StartupXLOG(void)
 	TrimCLOG();
 	TrimMultiXact();
 
-	/* Reload shared-memory state for prepared transactions */
+	/*
+	 * Reload shared-memory state for prepared transactions and foreign
+	 * prepared transactions.
+	 */
 	RecoverPreparedTransactions();
+	RecoverFdwXacts();
 
 	/*
 	 * Shutdown the recovery environment. This must occur after
@@ -9419,6 +9434,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 
 	/* We deliberately delay 2PC checkpointing as long as possible */
 	CheckPointTwoPhase(checkPointRedo);
+	CheckPointFdwXacts(checkPointRedo);
 }
 
 /*
@@ -9955,6 +9971,7 @@ XLogReportParameters(void)
 		max_worker_processes != ControlFile->max_worker_processes ||
 		max_wal_senders != ControlFile->max_wal_senders ||
 		max_prepared_xacts != ControlFile->max_prepared_xacts ||
+		max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts ||
 		max_locks_per_xact != ControlFile->max_locks_per_xact ||
 		track_commit_timestamp != ControlFile->track_commit_timestamp)
 	{
@@ -9974,6 +9991,7 @@ XLogReportParameters(void)
 			xlrec.max_worker_processes = max_worker_processes;
 			xlrec.max_wal_senders = max_wal_senders;
 			xlrec.max_prepared_xacts = max_prepared_xacts;
+			xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 			xlrec.max_locks_per_xact = max_locks_per_xact;
 			xlrec.wal_level = wal_level;
 			xlrec.wal_log_hints = wal_log_hints;
@@ -9992,6 +10010,7 @@ XLogReportParameters(void)
 		ControlFile->max_worker_processes = max_worker_processes;
 		ControlFile->max_wal_senders = max_wal_senders;
 		ControlFile->max_prepared_xacts = max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = max_locks_per_xact;
 		ControlFile->wal_level = wal_level;
 		ControlFile->wal_log_hints = wal_log_hints;
@@ -10198,6 +10217,7 @@ xlog_redo(XLogReaderState *record)
 			RunningTransactionsData running;
 
 			oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids);
+			oldestActiveXID = PrescanFdwXacts(oldestActiveXID);
 
 			/*
 			 * Construct a RunningTransactions snapshot representing a shut
@@ -10401,6 +10421,7 @@ xlog_redo(XLogReaderState *record)
 		ControlFile->max_worker_processes = xlrec.max_worker_processes;
 		ControlFile->max_wal_senders = xlrec.max_wal_senders;
 		ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts;
+		ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts;
 		ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact;
 		ControlFile->wal_level = xlrec.wal_level;
 		ControlFile->wal_log_hints = xlrec.wal_log_hints;
diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c
index 0c37fc1d53..b951c03d20 100644
--- a/src/backend/catalog/dependency.c
+++ b/src/backend/catalog/dependency.c
@@ -1470,6 +1470,10 @@ doDeletion(const ObjectAddress *object, int flags)
 			RemovePublicationRelById(object->objectId);
 			break;
 
+		case OCLASS_USER_MAPPING:
+			RemoveUserMappingById(object->objectId);
+			break;
+
 		case OCLASS_CAST:
 		case OCLASS_COLLATION:
 		case OCLASS_CONVERSION:
@@ -1485,7 +1489,6 @@ doDeletion(const ObjectAddress *object, int flags)
 		case OCLASS_TSTEMPLATE:
 		case OCLASS_FDW:
 		case OCLASS_FOREIGN_SERVER:
-		case OCLASS_USER_MAPPING:
 		case OCLASS_DEFACL:
 		case OCLASS_EVENT_TRIGGER:
 		case OCLASS_PUBLICATION:
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 999d984068..f36f05f5cc 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -402,6 +402,9 @@ CREATE VIEW pg_prepared_xacts AS
 CREATE VIEW pg_prepared_statements AS
     SELECT * FROM pg_prepared_statement() AS P;
 
+CREATE VIEW pg_foreign_xacts AS
+    SELECT * FROM pg_foreign_xacts() AS F;
+
 CREATE VIEW pg_seclabels AS
 SELECT
     l.objoid, l.classoid, l.objsubid,
diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c
index 2b159b60eb..25ad64639f 100644
--- a/src/backend/commands/dbcommands.c
+++ b/src/backend/commands/dbcommands.c
@@ -23,6 +23,7 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/fdwxact.h"
 #include "access/genam.h"
 #include "access/heapam.h"
 #include "access/htup_details.h"
@@ -821,6 +822,7 @@ dropdb(const char *dbname, bool missing_ok, bool force)
 	int			nslots,
 				nslots_active;
 	int			nsubscriptions;
+	int			nfdwxacts;
 
 	/*
 	 * Look up the target database's OID, and get exclusive lock on it. We
@@ -910,6 +912,18 @@ dropdb(const char *dbname, bool missing_ok, bool force)
 								  "There are %d subscriptions.",
 								  nsubscriptions, nsubscriptions)));
 
+	/*
+	 * Also check if there is foreign transaction associated with the target
+	 * database.
+	 */
+	if ((nfdwxacts = CountFdwXactsForDB(db_id)) > 0)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_IN_USE),
+				 errmsg("database \"%s\" is being used by foreign transaction resolution",
+						dbname),
+				 errdetail_plural("There is %d foreign transaction.",
+								  "There are %d foreign transactions.",
+								  nfdwxacts, nfdwxacts)));
 
 	/*
 	 * Attempt to terminate all existing connections to the target database if
diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c
index bc36311d38..533d5d59a1 100644
--- a/src/backend/commands/foreigncmds.c
+++ b/src/backend/commands/foreigncmds.c
@@ -13,6 +13,8 @@
  */
 #include "postgres.h"
 
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/htup_details.h"
 #include "access/reloptions.h"
 #include "access/table.h"
@@ -1060,6 +1062,49 @@ AlterForeignServer(AlterForeignServerStmt *stmt)
 	return address;
 }
 
+/*
+ * Drop foreign server
+ */
+Oid
+RemoveForeignServer(DropForeignServerStmt *stmt)
+{
+	Oid serverid;
+	ObjectAddress object;
+
+	serverid = GetSysCacheOid1(FOREIGNSERVERNAME, Anum_pg_foreign_server_oid,
+							   CStringGetDatum(stmt->servername));
+
+	if (!OidIsValid(serverid))
+	{
+		if (!stmt->missing_ok)
+			ereport(ERROR,
+					(errcode(ERRCODE_UNDEFINED_OBJECT),
+					 errmsg("server \"%s\" does not exist", stmt->servername)));
+
+		ereport(NOTICE,
+				errmsg("server \"%s\" does not exist, skipping",
+					   stmt->servername));
+		return InvalidOid;
+	}
+
+	/*
+	 * Only owner or a superuser can ALTER a SERVER.
+	 */
+	if (!pg_foreign_server_ownercheck(serverid, GetUserId()))
+		aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FOREIGN_SERVER,
+					   stmt->servername);
+
+	/* Do the deletion */
+	object.classId = ForeignServerRelationId;
+	object.objectId = serverid;
+	object.objectSubId = 0;
+	performDeletion(&object, stmt->behavior, 0);
+
+	/* Stop the foreign transaction resolver immediately */
+	StopFdwXactResolver(MyDatabaseId, serverid);
+
+	return serverid;
+}
 
 /*
  * Common routine to check permission for user-mapping-related DDL
@@ -1307,6 +1352,37 @@ AlterUserMapping(AlterUserMappingStmt *stmt)
 	return address;
 }
 
+/*
+ * Drop the given user mapping
+ */
+void
+RemoveUserMappingById(Oid umid)
+{
+	HeapTuple	tp;
+	Relation	rel;
+
+	rel = table_open(UserMappingRelationId, RowExclusiveLock);
+
+	tp = SearchSysCache1(USERMAPPINGOID, ObjectIdGetDatum(umid));
+
+	if (!HeapTupleIsValid(tp))
+		elog(ERROR, "cache lookup failed for user mapping %u", umid);
+
+	/*
+	 * We cannot drop the user mapping if there is a foreign prepared
+	 * transaction with this user mapping.
+	 */
+	if (CountFdwXactsForUserMapping(umid) > 0)
+		ereport(ERROR,
+				(errmsg("user mapping %u has unresolved prepared transaction",
+						umid)));
+
+	CatalogTupleDelete(rel, &tp->t_self);
+
+	ReleaseSysCache(tp);
+
+	table_close(rel, RowExclusiveLock);
+}
 
 /*
  * Drop user mapping
@@ -1374,6 +1450,7 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 
 	user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername);
 
+
 	/*
 	 * Do the deletion
 	 */
@@ -1386,7 +1463,6 @@ RemoveUserMapping(DropUserMappingStmt *stmt)
 	return umId;
 }
 
-
 /*
  * Create a foreign table
  * call after DefineRelation().
diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c
index f8eb4fa215..6ce76b2aec 100644
--- a/src/backend/foreign/foreign.c
+++ b/src/backend/foreign/foreign.c
@@ -332,6 +332,12 @@ GetFdwRoutine(Oid fdwhandler)
 	Assert((routine->CommitForeignTransaction && routine->RollbackForeignTransaction) ||
 		   (!routine->CommitForeignTransaction && !routine->RollbackForeignTransaction));
 
+	/* FDW supporting prepare API must support also commit and rollback APIs */
+	Assert((routine->PrepareForeignTransaction &&
+			routine->CommitForeignTransaction &&
+			routine->RollbackForeignTransaction) ||
+		   !routine->PrepareForeignTransaction);
+
 	return routine;
 }
 
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index bd87f23784..aad42485f5 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -4355,6 +4355,18 @@ _copyAlterForeignServerStmt(const AlterForeignServerStmt *from)
 	return newnode;
 }
 
+static DropForeignServerStmt *
+_copyDropForeignServerStmt(const DropForeignServerStmt *from)
+{
+	DropForeignServerStmt *newnode = makeNode(DropForeignServerStmt);
+
+	COPY_STRING_FIELD(servername);
+	COPY_SCALAR_FIELD(missing_ok);
+	COPY_SCALAR_FIELD(behavior);
+
+	return newnode;
+}
+
 static CreateUserMappingStmt *
 _copyCreateUserMappingStmt(const CreateUserMappingStmt *from)
 {
@@ -5604,6 +5616,9 @@ copyObjectImpl(const void *from)
 		case T_AlterForeignServerStmt:
 			retval = _copyAlterForeignServerStmt(from);
 			break;
+		case T_DropForeignServerStmt:
+			retval = _copyDropForeignServerStmt(from);
+			break;
 		case T_CreateUserMappingStmt:
 			retval = _copyCreateUserMappingStmt(from);
 			break;
diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c
index dba3e6b31e..e0b24d1147 100644
--- a/src/backend/nodes/equalfuncs.c
+++ b/src/backend/nodes/equalfuncs.c
@@ -1977,6 +1977,17 @@ _equalAlterForeignServerStmt(const AlterForeignServerStmt *a, const AlterForeign
 	return true;
 }
 
+static bool
+_equalDropForeignServerStmt(const DropForeignServerStmt *a,
+							const DropForeignServerStmt *b)
+{
+	COMPARE_STRING_FIELD(servername);
+	COMPARE_SCALAR_FIELD(missing_ok);
+	COMPARE_SCALAR_FIELD(behavior);
+
+	return true;
+}
+
 static bool
 _equalCreateUserMappingStmt(const CreateUserMappingStmt *a, const CreateUserMappingStmt *b)
 {
@@ -3595,6 +3606,9 @@ equal(const void *a, const void *b)
 		case T_AlterForeignServerStmt:
 			retval = _equalAlterForeignServerStmt(a, b);
 			break;
+		case T_DropForeignServerStmt:
+			retval = _equalDropForeignServerStmt(a, b);
+			break;
 		case T_CreateUserMappingStmt:
 			retval = _equalCreateUserMappingStmt(a, b);
 			break;
diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index eb24195438..85feb0a600 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -281,7 +281,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 		CreateAssertionStmt CreateTransformStmt CreateTrigStmt CreateEventTrigStmt
 		CreateUserStmt CreateUserMappingStmt CreateRoleStmt CreatePolicyStmt
 		CreatedbStmt DeclareCursorStmt DefineStmt DeleteStmt DiscardStmt DoStmt
-		DropOpClassStmt DropOpFamilyStmt DropStmt
+		DropOpClassStmt DropOpFamilyStmt DropForeignServerStmt DropStmt
 		DropCastStmt DropRoleStmt
 		DropdbStmt DropTableSpaceStmt
 		DropTransformStmt
@@ -980,6 +980,7 @@ stmt:
 			| DiscardStmt
 			| DoStmt
 			| DropCastStmt
+			| DropForeignServerStmt
 			| DropOpClassStmt
 			| DropOpFamilyStmt
 			| DropOwnedStmt
@@ -6339,6 +6340,7 @@ object_type_name:
 			drop_type_name							{ $$ = $1; }
 			| DATABASE								{ $$ = OBJECT_DATABASE; }
 			| ROLE									{ $$ = OBJECT_ROLE; }
+			| SERVER								{ $$ = OBJECT_FOREIGN_SERVER; }
 			| SUBSCRIPTION							{ $$ = OBJECT_SUBSCRIPTION; }
 			| TABLESPACE							{ $$ = OBJECT_TABLESPACE; }
 		;
@@ -6351,7 +6353,6 @@ drop_type_name:
 			| opt_procedural LANGUAGE				{ $$ = OBJECT_LANGUAGE; }
 			| PUBLICATION							{ $$ = OBJECT_PUBLICATION; }
 			| SCHEMA								{ $$ = OBJECT_SCHEMA; }
-			| SERVER								{ $$ = OBJECT_FOREIGN_SERVER; }
 		;
 
 /* object types attached to a table */
@@ -9798,6 +9799,30 @@ DropSubscriptionStmt: DROP SUBSCRIPTION name opt_drop_behavior
 				}
 		;
 
+/*****************************************************************************
+ *
+ * DROP SERVER [ IF EXISTS ] name
+ *
+ *****************************************************************************/
+
+DropForeignServerStmt: DROP SERVER name opt_drop_behavior
+				{
+					DropForeignServerStmt *n = makeNode(DropForeignServerStmt);
+					n->servername = $3;
+					n->missing_ok = false;
+					n->behavior = $4;
+					$$ = (Node *) n;
+				}
+				|  DROP SERVER IF_P EXISTS name opt_drop_behavior
+				{
+					DropForeignServerStmt *n = makeNode(DropForeignServerStmt);
+					n->servername = $5;
+					n->missing_ok = true;
+					n->behavior = $6;
+					$$ = (Node *) n;
+				}
+		;
+
 /*****************************************************************************
  *
  *		QUERY:	Define Rewrite Rule
diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index c40410d73e..89d0219cad 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -13,6 +13,8 @@
 #include "postgres.h"
 
 #include "access/parallel.h"
+#include "access/fdwxact_resolver.h"
+#include "access/fdwxact_launcher.h"
 #include "libpq/pqsignal.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -128,6 +130,12 @@ static const struct
 	},
 	{
 		"ApplyWorkerMain", ApplyWorkerMain
+	},
+	{
+		"FdwXactResolverMain", FdwXactResolverMain
+	},
+	{
+		"FdwXactLauncherMain", FdwXactLauncherMain
 	}
 };
 
diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c
index 5a050898fe..d5afe2382c 100644
--- a/src/backend/postmaster/postmaster.c
+++ b/src/backend/postmaster/postmaster.c
@@ -93,6 +93,7 @@
 #include <pthread.h>
 #endif
 
+#include "access/fdwxact_launcher.h"
 #include "access/transam.h"
 #include "access/xlog.h"
 #include "bootstrap/bootstrap.h"
@@ -925,6 +926,9 @@ PostmasterMain(int argc, char *argv[])
 	if (max_wal_senders > 0 && wal_level == WAL_LEVEL_MINIMAL)
 		ereport(ERROR,
 				(errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\"")));
+	if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers <= 0)
+		ereport(ERROR,
+				(errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0")));
 
 	/*
 	 * Other one-time internal sanity checks can go here, if they are fast.
@@ -990,12 +994,13 @@ PostmasterMain(int argc, char *argv[])
 	LocalProcessControlFile(false);
 
 	/*
-	 * Register the apply launcher.  Since it registers a background worker,
-	 * it needs to be called before InitializeMaxBackends(), and it's probably
-	 * a good idea to call it before any modules had chance to take the
-	 * background worker slots.
+	 * Register the apply launcher and foreign transaction launcher.  Since
+	 * it registers a background worker, it needs to be called before
+	 * InitializeMaxBackends(), and it's probably a good idea to call it
+	 * before any modules had chance to take the background worker slots.
 	 */
 	ApplyLauncherRegister();
+	FdwXactLauncherRegister();
 
 	/*
 	 * process any libraries that should be preloaded at postmaster start
diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index 453efc51e1..eac334c5f6 100644
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -179,6 +179,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor
 		case RM_COMMIT_TS_ID:
 		case RM_REPLORIGIN_ID:
 		case RM_GENERIC_ID:
+		case RM_FDWXACT_ID:
 			/* just deal with xid, and done */
 			ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record),
 									buf.origptr);
diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c
index bdbc9ef844..c4f4ca8c65 100644
--- a/src/backend/replication/syncrep.c
+++ b/src/backend/replication/syncrep.c
@@ -143,13 +143,16 @@ static bool SyncRepQueueIsOrderedByLSN(int mode);
  * represents a commit record.  If it doesn't, then we wait only for the WAL
  * to be flushed if synchronous_commit is set to the higher level of
  * remote_apply, because only commit records provide apply feedback.
+ *
+ * Return true if the waits is cancaled by an interruption.
  */
-void
+bool
 SyncRepWaitForLSN(XLogRecPtr lsn, bool commit)
 {
 	char	   *new_status = NULL;
 	const char *old_status;
 	int			mode;
+	bool		canceled = false;
 
 	/*
 	 * This should be called while holding interrupts during a transaction
@@ -173,7 +176,7 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit)
 	 */
 	if (!SyncRepRequested() ||
 		!((volatile WalSndCtlData *) WalSndCtl)->sync_standbys_defined)
-		return;
+		return false;
 
 	/* Cap the level for anything other than commit to remote flush only. */
 	if (commit)
@@ -199,7 +202,7 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit)
 		lsn <= WalSndCtl->lsn[mode])
 	{
 		LWLockRelease(SyncRepLock);
-		return;
+		return false;
 	}
 
 	/*
@@ -269,6 +272,7 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit)
 					 errdetail("The transaction has already committed locally, but might not have been replicated to the standby.")));
 			whereToSendOutput = DestNone;
 			SyncRepCancelWait();
+			canceled = true;
 			break;
 		}
 
@@ -285,6 +289,7 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit)
 					(errmsg("canceling wait for synchronous replication due to user request"),
 					 errdetail("The transaction has already committed locally, but might not have been replicated to the standby.")));
 			SyncRepCancelWait();
+			canceled = true;
 			break;
 		}
 
@@ -304,6 +309,7 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit)
 			ProcDiePending = true;
 			whereToSendOutput = DestNone;
 			SyncRepCancelWait();
+			canceled = true;
 			break;
 		}
 	}
@@ -327,6 +333,8 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit)
 		set_ps_display(new_status);
 		pfree(new_status);
 	}
+
+	return canceled;
 }
 
 /*
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index 3e4ec53a97..47179c37a4 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -16,6 +16,8 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
+#include "access/fdwxact_launcher.h"
 #include "access/heapam.h"
 #include "access/multixact.h"
 #include "access/nbtree.h"
@@ -150,6 +152,8 @@ CreateSharedMemoryAndSemaphores(void)
 		size = add_size(size, BTreeShmemSize());
 		size = add_size(size, SyncScanShmemSize());
 		size = add_size(size, AsyncShmemSize());
+		size = add_size(size, FdwXactShmemSize());
+		size = add_size(size, FdwXactLauncherShmemSize());
 #ifdef EXEC_BACKEND
 		size = add_size(size, ShmemBackendArraySize());
 #endif
@@ -269,6 +273,8 @@ CreateSharedMemoryAndSemaphores(void)
 	BTreeShmemInit();
 	SyncScanShmemInit();
 	AsyncShmemInit();
+	FdwXactShmemInit();
+	FdwXactLauncherShmemInit();
 
 #ifdef EXEC_BACKEND
 
diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c
index 793df973b4..417355f795 100644
--- a/src/backend/storage/ipc/procarray.c
+++ b/src/backend/storage/ipc/procarray.c
@@ -96,6 +96,8 @@ typedef struct ProcArrayStruct
 	TransactionId replication_slot_xmin;
 	/* oldest catalog xmin of any replication slot */
 	TransactionId replication_slot_catalog_xmin;
+	/* local transaction id of oldest unresolved distributed transaction */
+	TransactionId fdwxact_unresolved_xmin;
 
 	/* indexes into allProcs[], has PROCARRAY_MAXPROCS entries */
 	int			pgprocnos[FLEXIBLE_ARRAY_MEMBER];
@@ -187,11 +189,13 @@ typedef struct ComputeXidHorizonsResult
 	FullTransactionId latest_completed;
 
 	/*
-	 * The same for procArray->replication_slot_xmin and.
-	 * procArray->replication_slot_catalog_xmin.
+	 * The same for procArray->replication_slot_xmin,
+	 * procArray->replication_slot_catalog_xmin, and
+	 * procArray->fdwxact_unresolved_xmin.
 	 */
 	TransactionId slot_xmin;
 	TransactionId slot_catalog_xmin;
+	TransactionId fdwxact_unresolved_xmin;
 
 	/*
 	 * Oldest xid that any backend might still consider running. This needs to
@@ -210,8 +214,9 @@ typedef struct ComputeXidHorizonsResult
 	 * Oldest xid for which deleted tuples need to be retained in shared
 	 * tables.
 	 *
-	 * This includes the effects of replication slots. If that's not desired,
-	 * look at shared_oldest_nonremovable_raw;
+	 * This includes the effects of replication slots as unresolved
+	 * foreign transactions. If that's not desired, look at
+	 * shared_oldest_nonremovable_raw;
 	 */
 	TransactionId shared_oldest_nonremovable;
 
@@ -418,6 +423,7 @@ CreateSharedProcArray(void)
 		procArray->lastOverflowedXid = InvalidTransactionId;
 		procArray->replication_slot_xmin = InvalidTransactionId;
 		procArray->replication_slot_catalog_xmin = InvalidTransactionId;
+		procArray->fdwxact_unresolved_xmin = InvalidTransactionId;
 		ShmemVariableCache->xactCompletionCount = 1;
 	}
 
@@ -1741,6 +1747,7 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h)
 	 */
 	h->slot_xmin = procArray->replication_slot_xmin;
 	h->slot_catalog_xmin = procArray->replication_slot_catalog_xmin;
+	h->fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin;
 
 	for (int index = 0; index < arrayP->numProcs; index++)
 	{
@@ -1888,6 +1895,15 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h)
 	h->data_oldest_nonremovable =
 		TransactionIdOlder(h->data_oldest_nonremovable, h->slot_xmin);
 
+	/*
+	 * Check whether there are unresolved distributed transaction requiring
+	 * an older xmin.
+	 */
+	h->shared_oldest_nonremovable =
+		TransactionIdOlder(h->shared_oldest_nonremovable, h->fdwxact_unresolved_xmin);
+	h->data_oldest_nonremovable =
+		TransactionIdOlder(h->data_oldest_nonremovable, h->fdwxact_unresolved_xmin);
+
 	/*
 	 * The only difference between catalog / data horizons is that the slot's
 	 * catalog xmin is applied to the catalog one (so catalogs can be accessed
@@ -1947,6 +1963,9 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h)
 	Assert(!TransactionIdIsValid(h->slot_catalog_xmin) ||
 		   TransactionIdPrecedesOrEquals(h->oldest_considered_running,
 										 h->slot_catalog_xmin));
+	Assert(!TransactionIdIsValid(h->fdwxact_unresolved_xmin) ||
+		   TransactionIdPrecedesOrEquals(h->oldest_considered_running,
+										 h->fdwxact_unresolved_xmin));
 
 	/* update approximate horizons with the computed horizons */
 	GlobalVisUpdateApply(h);
@@ -3859,6 +3878,21 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 	LWLockRelease(ProcArrayLock);
 }
 
+/*
+ * ProcArraySetFdwXactUnresolvedXmin
+ *
+ * Install limits to future computations of the xmin horizon to prevent
+ * vacuum clog from affected transactions needed by resolving distributed
+ * transaction.
+ */
+void
+ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin)
+{
+	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+	procArray->fdwxact_unresolved_xmin = xmin;
+	LWLockRelease(ProcArrayLock);
+}
+
 /*
  * XidCacheRemoveRunningXids
  *
diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt
index 6c7cf6c295..a297c746cd 100644
--- a/src/backend/storage/lmgr/lwlocknames.txt
+++ b/src/backend/storage/lmgr/lwlocknames.txt
@@ -53,3 +53,5 @@ XactTruncationLock					44
 # 45 was XactTruncationLock until removal of BackendRandomLock
 WrapLimitsVacuumLock				46
 NotifyQueueTailLock					47
+FdwXactLock							48
+FdwXactResolverLock					49
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 8cea10c901..929061a6bb 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -36,6 +36,8 @@
 #include "rusagestub.h"
 #endif
 
+#include "access/fdwxact_launcher.h"
+#include "access/fdwxact_resolver.h"
 #include "access/parallel.h"
 #include "access/printtup.h"
 #include "access/xact.h"
@@ -3164,6 +3166,20 @@ ProcessInterrupts(void)
 			 */
 			proc_exit(1);
 		}
+		else if (IsFdwXactResolver())
+			ereport(FATAL,
+					(errcode(ERRCODE_ADMIN_SHUTDOWN),
+					 errmsg("terminating foreign transaction resolver due to administrator command")));
+		else if (IsFdwXactLauncher())
+		{
+			ereport(DEBUG1,
+					(errmsg_internal("foreign transaction launcher shutting down")));
+			/*
+			 * The foreign transaction launcher can be stopped at any time.
+			 * Use exit status 1 so the background worker is restarted.
+			 */
+			proc_exit(1);
+		}
 		else if (RecoveryConflictPending && RecoveryConflictRetryable)
 		{
 			pgstat_report_recovery_conflict(RecoveryConflictReason);
diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c
index 7a2da9dab4..95cf27f7cf 100644
--- a/src/backend/tcop/utility.c
+++ b/src/backend/tcop/utility.c
@@ -197,6 +197,7 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree)
 		case T_CreateUserMappingStmt:
 		case T_CreatedbStmt:
 		case T_DefineStmt:
+		case T_DropForeignServerStmt:
 		case T_DropOwnedStmt:
 		case T_DropRoleStmt:
 		case T_DropStmt:
@@ -1567,6 +1568,10 @@ ProcessUtilitySlow(ParseState *pstate,
 				address = AlterForeignServer((AlterForeignServerStmt *) parsetree);
 				break;
 
+			case T_DropForeignServerStmt:
+				RemoveForeignServer((DropForeignServerStmt *) parsetree);
+				break;
+
 			case T_CreateUserMappingStmt:
 				address = CreateUserMapping((CreateUserMappingStmt *) parsetree);
 				break;
@@ -2473,6 +2478,10 @@ CreateCommandTag(Node *parsetree)
 			tag = CMDTAG_ALTER_SERVER;
 			break;
 
+		case T_DropForeignServerStmt:
+			tag = CMDTAG_DROP_SERVER;
+			break;
+
 		case T_CreateUserMappingStmt:
 			tag = CMDTAG_CREATE_USER_MAPPING;
 			break;
@@ -2577,9 +2586,6 @@ CreateCommandTag(Node *parsetree)
 				case OBJECT_FDW:
 					tag = CMDTAG_DROP_FOREIGN_DATA_WRAPPER;
 					break;
-				case OBJECT_FOREIGN_SERVER:
-					tag = CMDTAG_DROP_SERVER;
-					break;
 				case OBJECT_OPCLASS:
 					tag = CMDTAG_DROP_OPERATOR_CLASS;
 					break;
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index 6baf67740c..429a50c591 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -726,6 +726,21 @@ pgstat_get_wait_io(WaitEventIO w)
 		case WAIT_EVENT_LOGICAL_SUBXACT_WRITE:
 			event_name = "LogicalSubxactWrite";
 			break;
+		case WAIT_EVENT_FDWXACT_FILE_READ:
+			event_name = "FdwXactFileRead";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_SYNC:
+			event_name = "FdwXactFileSync";
+			break;
+		case WAIT_EVENT_FDWXACT_FILE_WRITE:
+			event_name = "FdwXactFileWrite";
+			break;
+		case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN:
+			event_name = "FdwXactLauncherMain";
+			break;
+		case WAIT_EVENT_FDWXACT_RESOLVER_MAIN:
+			event_name = "FdwXactResolverMain";
+			break;
 
 			/* no default case, so that compiler will warn */
 	}
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 480e8cd199..65815ec047 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -33,6 +33,7 @@
 #include <unistd.h>
 
 #include "access/commit_ts.h"
+#include "access/fdwxact.h"
 #include "access/gin.h"
 #include "access/rmgr.h"
 #include "access/tableam.h"
@@ -802,6 +803,10 @@ const char *const config_group_names[] =
 	gettext_noop("Client Connection Defaults / Other Defaults"),
 	/* LOCK_MANAGEMENT */
 	gettext_noop("Lock Management"),
+	/* FOREIGN_TRANSACTION */
+	gettext_noop("Foreign Transaction"),
+	/* FOREIGN_TRANSACTION_RESOLVER */
+	gettext_noop("Foreign Transaction / Resolver"),
 	/* COMPAT_OPTIONS_PREVIOUS */
 	gettext_noop("Version and Platform Compatibility / Previous PostgreSQL Versions"),
 	/* COMPAT_OPTIONS_CLIENT */
@@ -2538,6 +2543,49 @@ static struct config_int ConfigureNamesInt[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."),
+			NULL
+		},
+		&max_prepared_foreign_xacts,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolver_timeout", PGC_SIGHUP, FOREIGN_TRANSACTION_RESOLVER,
+			gettext_noop("Sets the maximum time to wait for foreign transaction resolution."),
+			NULL,
+			GUC_UNIT_MS
+		},
+		&foreign_xact_resolver_timeout,
+		60 * 1000, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM,
+			gettext_noop("Maximum number of foreign transaction resolution processes."),
+			NULL
+		},
+		&max_foreign_xact_resolvers,
+		0, 0, INT_MAX,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FOREIGN_TRANSACTION_RESOLVER,
+		 gettext_noop("Sets the time to wait before retrying to resolve foreign transaction "
+					  "after a failed attempt."),
+		 NULL,
+		 GUC_UNIT_MS
+		},
+		&foreign_xact_resolution_retry_interval,
+		5000, 1, INT_MAX,
+		NULL, NULL, NULL
+	},
+
 #ifdef LOCK_DEBUG
 	{
 		{"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS,
diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample
index b696abfe54..9cc35c7109 100644
--- a/src/backend/utils/misc/postgresql.conf.sample
+++ b/src/backend/utils/misc/postgresql.conf.sample
@@ -132,6 +132,8 @@
 #temp_buffers = 8MB			# min 800kB
 #max_prepared_transactions = 0		# zero disables the feature
 					# (change requires restart)
+#max_prepared_foreign_transactions = 0	# zero disables the feature
+					# (change requires restart)
 # Caution: it is not advisable to set max_prepared_transactions nonzero unless
 # you actively intend to use prepared transactions.
 #work_mem = 4MB				# min 64kB
@@ -744,6 +746,18 @@
 #max_pred_locks_per_page = 2            # min 0
 
 
+#------------------------------------------------------------------------------
+# FOREIGN TRANSACTION
+#------------------------------------------------------------------------------
+
+#max_foreign_transaction_resolvers = 0		# max number of resolver process
+						# (change requires restart)
+#foreign_transaction_resolver_timeout = 60s	# in milliseconds; 0 disables
+#foreign_transaction_resolution_retry_interval = 5s	# time to wait before
+							# retrying to resolve
+							# foreign transactions
+							# after a failed attempt
+
 #------------------------------------------------------------------------------
 # VERSION AND PLATFORM COMPATIBILITY
 #------------------------------------------------------------------------------
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 152d21e88b..735e4084b3 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -207,6 +207,7 @@ static const char *const subdirs[] = {
 	"pg_wal/archive_status",
 	"pg_commit_ts",
 	"pg_dynshmem",
+	"pg_fdwxact",
 	"pg_notify",
 	"pg_serial",
 	"pg_snapshots",
diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c
index f911f98d94..49d47c2ee7 100644
--- a/src/bin/pg_controldata/pg_controldata.c
+++ b/src/bin/pg_controldata/pg_controldata.c
@@ -296,6 +296,8 @@ main(int argc, char *argv[])
 		   ControlFile->max_wal_senders);
 	printf(_("max_prepared_xacts setting:           %d\n"),
 		   ControlFile->max_prepared_xacts);
+	printf(_("max_prepared_foreign_transactions setting:   %d\n"),
+		   ControlFile->max_prepared_foreign_xacts);
 	printf(_("max_locks_per_xact setting:           %d\n"),
 		   ControlFile->max_locks_per_xact);
 	printf(_("track_commit_timestamp setting:       %s\n"),
diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c
index 805dafef07..dd70a0f8a2 100644
--- a/src/bin/pg_resetwal/pg_resetwal.c
+++ b/src/bin/pg_resetwal/pg_resetwal.c
@@ -710,6 +710,7 @@ GuessControlValues(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	ControlFile.maxAlign = MAXIMUM_ALIGNOF;
@@ -914,6 +915,7 @@ RewriteControlFile(void)
 	ControlFile.max_wal_senders = 10;
 	ControlFile.max_worker_processes = 8;
 	ControlFile.max_prepared_xacts = 0;
+	ControlFile.max_prepared_foreign_xacts = 0;
 	ControlFile.max_locks_per_xact = 64;
 
 	/* The control file gets flushed here. */
diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c
index 852d8ca4b1..b616cea347 100644
--- a/src/bin/pg_waldump/rmgrdesc.c
+++ b/src/bin/pg_waldump/rmgrdesc.c
@@ -11,6 +11,7 @@
 #include "access/brin_xlog.h"
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/fdwxact_xlog.h"
 #include "access/generic_xlog.h"
 #include "access/ginxlog.h"
 #include "access/gistxlog.h"
diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h
index 1d4a285c75..85854864b9 100644
--- a/src/include/access/fdwxact.h
+++ b/src/include/access/fdwxact.h
@@ -1,7 +1,7 @@
 /*
  * fdwxact.h
  *
- * PostgreSQL global transaction manager
+ * PostgreSQL foreign transaction manager definitions
  *
  * Portions Copyright (c) 2021, PostgreSQL Global Development Group
  *
@@ -11,13 +11,80 @@
 #define FDWXACT_H
 
 #include "access/xact.h"
+#include "access/fdwxact_xlog.h"
 #include "foreign/foreign.h"
+#include "storage/proc.h"
+#include "storage/shmem.h"
+#include "storage/s_lock.h"
 
 /* Flag passed to FDW transaction management APIs */
 #define FDWXACT_FLAG_ONEPHASE		0x01	/* transaction can commit/rollback
 											 * without preparation */
 #define FDWXACT_FLAG_PARALLEL_WORKER	0x02	/* is parallel worker? */
 
+/* Enum to track the status of foreign transaction */
+typedef enum
+{
+	FDWXACT_STATUS_INVALID = 0,
+	FDWXACT_STATUS_PREPARING,	/* foreign transaction is being prepared */
+	FDWXACT_STATUS_PREPARED,	/* foreign transaction is prepared */
+	FDWXACT_STATUS_COMMITTING,	/* foreign prepared transaction is committed */
+	FDWXACT_STATUS_ABORTING		/* foreign prepared transaction is aborted */
+} FdwXactStatus;
+
+/*
+ * Shared memory state of a single foreign transaction.
+ */
+typedef struct FdwXactStateData *FdwXactState;
+typedef struct FdwXactStateData
+{
+	FdwXactState		fdwxact_free_next;	/* Next free FdwXactState entry */
+
+	/* Information relevant with foreign transaction */
+	FdwXactStateOnDiskData data;
+
+	/* Foreign transaction status */
+	FdwXactStatus status;
+	slock_t		mutex;			/* protect the above field */
+
+	/*
+	 * Note that we need to keep track of two LSNs for each FdwXactState. We keep
+	 * track of the start LSN because this is the address we must use to read
+	 * state data back from WAL when committing a FdwXactState. We keep track of
+	 * the end LSN because that is the LSN we need to wait for prior to
+	 * commit.
+	 */
+	XLogRecPtr	insert_start_lsn;	/* XLOG offset of inserting this entry
+									 * start */
+	XLogRecPtr	insert_end_lsn; /* XLOG offset of inserting this entry end */
+
+	bool		valid;			/* has the entry been complete and written to
+								 * file? */
+	BackendId	locking_backend;	/* backend currently working on the fdw xact */
+	bool		ondisk;			/* true if prepare state file is on disk */
+	bool		inredo;			/* true if entry was added via xlog_redo */
+} FdwXactStateData;
+
+/*
+ * Shared memory layout for maintaining foreign prepared transaction entries.
+ * Adding or removing FdwXactState entry needs to hold FdwXactLock in exclusive mode,
+ * and iterating fdwXacts needs that in shared mode.
+ */
+typedef struct
+{
+	/* Head of linked list of free FdwXactStateData structs */
+	FdwXactState	free_fdwxacts;
+
+	/* Number of valid foreign transaction entries */
+	int	num_xacts;
+
+	/* Upto max_prepared_foreign_xacts entries in the array */
+	FdwXactState	xacts[FLEXIBLE_ARRAY_MEMBER];	/* Variable length array */
+} FdwXactCtlData;
+
+/* Pointer to the shared memory holding the foreign transactions data */
+FdwXactCtlData *FdwXactCtl;
+
 /* State data for foreign transaction resolution, passed to FDW callbacks */
 typedef struct FdwXactInfo
 {
@@ -25,10 +92,29 @@ typedef struct FdwXactInfo
 	UserMapping		*usermapping;
 
 	int	flags;			/* OR of FDWXACT_FLAG_xx flags */
+	char   *identifier;
 } FdwXactInfo;
 
+/* GUC parameters */
+extern int	max_prepared_foreign_xacts;
+extern int	max_foreign_xact_resolvers;
+extern int	foreign_xact_resolution_retry_interval;
+extern int	foreign_xact_resolver_timeout;
+
 /* Function declarations */
+extern void PreCommit_FdwXact(bool is_parallel_worker);
 extern void AtEOXact_FdwXact(bool isCommit, bool is_parallel_worker);
+extern Size FdwXactShmemSize(void);
+extern void FdwXactShmemInit(void);
 extern void AtPrepare_FdwXact(void);
+extern bool FdwXactIsForeignTwophaseCommitRequired(void);
+extern int CountFdwXactsForUserMapping(Oid umid);
+extern int CountFdwXactsForDB(Oid dbid);
+extern void FdwXactLaunchResolversForXid(TransactionId xid);
+extern void CheckPointFdwXacts(XLogRecPtr redo_horizon);
+extern void ResolveOneFdwXact(FdwXactState fdwxact);
+extern void RestoreFdwXactData(void);
+extern void RecoverFdwXacts(void);
+extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid);
 
 #endif /* FDWXACT_H */
diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h
new file mode 100644
index 0000000000..6381aa6e55
--- /dev/null
+++ b/src/include/access/fdwxact_launcher.h
@@ -0,0 +1,29 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_launcher.h
+ *	  PostgreSQL foreign transaction launcher definitions
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_launcher.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FDWXACT_LAUNCHER_H
+#define FDWXACT_LAUNCHER_H
+
+#include "access/fdwxact.h"
+#include "access/resolver_internal.h"
+
+extern void FdwXactLauncherRegister(void);
+extern void FdwXactLauncherMain(Datum main_arg);
+extern void RequestToLaunchFdwXactResolver(void);
+extern void LaunchOrWakeupFdwXactResolver(List *serveroids_orig);
+extern Size FdwXactLauncherShmemSize(void);
+extern void FdwXactLauncherShmemInit(void);
+extern bool IsFdwXactLauncher(void);
+extern void StopFdwXactResolver(Oid dbid, Oid serverid);
+
+
+#endif							/* FDWXACT_LAUNCHER_H */
diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h
new file mode 100644
index 0000000000..9301ada5bb
--- /dev/null
+++ b/src/include/access/fdwxact_resolver.h
@@ -0,0 +1,22 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_resolver.h
+ *	  PostgreSQL foreign transaction resolver definitions
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_resolver.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_RESOLVER_H
+#define FDWXACT_RESOLVER_H
+
+#include "access/fdwxact.h"
+
+extern void FdwXactResolverMain(Datum main_arg);
+extern bool IsFdwXactResolver(void);
+
+extern int	foreign_xact_resolver_timeout;
+
+#endif							/* FDWXACT_RESOLVER_H */
diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h
new file mode 100644
index 0000000000..a1a10b71b2
--- /dev/null
+++ b/src/include/access/fdwxact_xlog.h
@@ -0,0 +1,49 @@
+/*-------------------------------------------------------------------------
+ *
+ * fdwxact_xlog.h
+ *	  Foreign transaction XLOG definitions.
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/include/access/fdwxact_xlog.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FDWXACT_XLOG_H
+#define FDWXACT_XLOG_H
+
+#include "access/xlogreader.h"
+#include "lib/stringinfo.h"
+
+/* Info types for logs related to FDW transactions */
+#define XLOG_FDWXACT_INSERT	0x00
+#define XLOG_FDWXACT_REMOVE	0x10
+
+/* Maximum length of the prepared transaction id, borrowed from twophase.c */
+#define FDWXACT_ID_MAX_LEN 200
+
+/*
+ * On disk file structure, also used to WAL
+ */
+typedef struct
+{
+	TransactionId xid;
+	Oid		dbid;
+	Oid		umid;
+	Oid		serverid;
+	Oid		owner;
+	char	identifier[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */
+} FdwXactStateOnDiskData;
+
+typedef struct xl_fdwxact_remove
+{
+	TransactionId xid;
+	Oid		umid;
+	bool	force;
+} xl_fdwxact_remove;
+
+extern void fdwxact_redo(XLogReaderState *record);
+extern void fdwxact_desc(StringInfo buf, XLogReaderState *record);
+extern const char *fdwxact_identify(uint8 info);
+
+#endif							/* FDWXACT_XLOG_H */
diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h
new file mode 100644
index 0000000000..89fc6a5653
--- /dev/null
+++ b/src/include/access/resolver_internal.h
@@ -0,0 +1,59 @@
+/*-------------------------------------------------------------------------
+ *
+ * resolver_internal.h
+ *	  Internal headers shared by fdwxact resolvers.
+ *
+ * Portions Copyright (c) 2021, PostgreSQL Global Development Group
+ *
+ * src/include/access/resolver_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef RESOLVER_INTERNAL_H
+#define RESOLVER_INTERNAL_H
+
+#include "storage/latch.h"
+#include "storage/shmem.h"
+#include "storage/spin.h"
+#include "utils/timestamp.h"
+
+/*
+ * Each foreign transaction resolver has a FdwXactResolver struct in
+ * shared memory.  This struct is protected by FdwXactResolverLock.
+ */
+typedef struct FdwXactResolver
+{
+	pid_t		pid;			/* this resolver's PID, or 0 if not active */
+	Oid			dbid;
+	Oid			serverid;
+
+	/* Indicates if this slot is used of free */
+	bool		in_use;
+
+	/* Protect shared variables shown above */
+	slock_t		mutex;
+
+	/*
+	 * Pointer to the resolver's patch. Used by backends to wake up this
+	 * resolver when it has work to do. NULL if the resolver isn't active.
+	 */
+	Latch	   *latch;
+} FdwXactResolver;
+
+/* There is one FdwXactResolverCtlData struct for the whole database cluster */
+typedef struct FdwXactResolverCtlData
+{
+	/* Supervisor process and latch */
+	pid_t		launcher_pid;
+	Latch	   *launcher_latch;
+
+	FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER];
+} FdwXactResolverCtlData;
+#define SizeOfFdwXactResolverCtlData \
+	(offsetof(FdwXactResolverCtlData, resolvers) + sizeof(FdwXactResolver))
+
+extern FdwXactResolverCtlData *FdwXactResolverCtl;
+extern FdwXactResolver *MyFdwXactResolver;
+
+#endif							/* RESOLVER_INTERNAL_H */
diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h
index f582cf535f..5ab1f57212 100644
--- a/src/include/access/rmgrlist.h
+++ b/src/include/access/rmgrlist.h
@@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i
 PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL)
 PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask)
 PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL)
+PG_RMGR(RM_FDWXACT_ID, "Fdw Transaction", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL)
diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h
index 91786da784..3d35f89ae0 100644
--- a/src/include/access/twophase.h
+++ b/src/include/access/twophase.h
@@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held);
 extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held);
+extern bool	TwoPhaseExists(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 										 TimestampTz prepared_at,
diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index 3b5eceff65..21fa835687 100644
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -236,6 +236,7 @@ typedef struct xl_parameter_change
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	int			wal_level;
 	bool		wal_log_hints;
diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index e3f48158ce..5673ec7299 100644
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -179,6 +179,7 @@ typedef struct ControlFileData
 	int			max_worker_processes;
 	int			max_wal_senders;
 	int			max_prepared_xacts;
+	int			max_prepared_foreign_xacts;
 	int			max_locks_per_xact;
 	bool		track_commit_timestamp;
 
diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat
index fde251fa4f..7569f4cdbd 100644
--- a/src/include/catalog/pg_proc.dat
+++ b/src/include/catalog/pg_proc.dat
@@ -6123,6 +6123,24 @@
   proargnames => '{type,object_names,object_args,classid,objid,objsubid}',
   prosrc => 'pg_get_object_address' },
 
+{ oid => '9706', descr => 'view foreign transactions',
+  proname => 'pg_foreign_xacts', prorows => '100', proretset => 't',
+  provolatile => 'v', prorettype => 'record', proargtypes => '',
+  proallargtypes => '{xid,oid,oid,oid,text,text,int4}',
+  proargmodes => '{o,o,o,o,o,o,o}',
+  proargnames => '{xid,umid,ownerid,dbid,state,identifier,locker_pid}',
+  prosrc => 'pg_foreign_xacts' },
+{ oid => '9707', descr => 'remove foreign transaction without resolution',
+  proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid',
+  proargnames => '{xid,umid}',
+  prosrc => 'pg_remove_foreign_xact' },
+{ oid => '9708', descr => 'resolve one foreign transaction',
+  proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'xid oid',
+  proargnames => '{xid,umid}',
+  prosrc => 'pg_resolve_foreign_xact' },
+
 { oid => '2079', descr => 'is table visible in search path?',
   proname => 'pg_table_is_visible', procost => '10', provolatile => 's',
   prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' },
@@ -6243,6 +6261,11 @@
   proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn',
   prosrc => 'pg_walfile_name' },
 
+{ oid => '9709',
+  descr => 'stop a running foreign transaction resolver process',
+  proname => 'pg_stop_foreign_xact_resolver', provolatile => 'v', prorettype => 'bool',
+  proargtypes => 'text', prosrc => 'pg_stop_foreign_xact_resolver'},
+
 { oid => '3165', descr => 'difference in bytes, given two wal locations',
   proname => 'pg_wal_lsn_diff', prorettype => 'numeric',
   proargtypes => 'pg_lsn pg_lsn', prosrc => 'pg_wal_lsn_diff' },
diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h
index 42bf1c7519..a38a451afd 100644
--- a/src/include/commands/defrem.h
+++ b/src/include/commands/defrem.h
@@ -125,9 +125,11 @@ extern ObjectAddress CreateForeignDataWrapper(CreateFdwStmt *stmt);
 extern ObjectAddress AlterForeignDataWrapper(AlterFdwStmt *stmt);
 extern ObjectAddress CreateForeignServer(CreateForeignServerStmt *stmt);
 extern ObjectAddress AlterForeignServer(AlterForeignServerStmt *stmt);
+extern Oid RemoveForeignServer(DropForeignServerStmt *stmt);
 extern ObjectAddress CreateUserMapping(CreateUserMappingStmt *stmt);
 extern ObjectAddress AlterUserMapping(AlterUserMappingStmt *stmt);
 extern Oid	RemoveUserMapping(DropUserMappingStmt *stmt);
+extern void RemoveUserMappingById(Oid umid);
 extern void CreateForeignTable(CreateForeignTableStmt *stmt, Oid relid);
 extern void ImportForeignSchema(ImportForeignSchemaStmt *stmt);
 extern Datum transformGenericOptions(Oid catalogId,
diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h
index c3539a4d73..5338f4f2d9 100644
--- a/src/include/foreign/fdwapi.h
+++ b/src/include/foreign/fdwapi.h
@@ -192,6 +192,7 @@ typedef void (*ForeignAsyncConfigureWait_function) (AsyncRequest *areq);
 
 typedef void (*ForeignAsyncNotify_function) (AsyncRequest *areq);
 
+typedef void (*PrepareForeignTransaction_function) (FdwXactInfo *finfo);
 typedef void (*CommitForeignTransaction_function) (FdwXactInfo *finfo);
 typedef void (*RollbackForeignTransaction_function) (FdwXactInfo *finfo);
 
@@ -287,6 +288,7 @@ typedef struct FdwRoutine
 	/* Support functions for transaction management */
 	CommitForeignTransaction_function CommitForeignTransaction;
 	RollbackForeignTransaction_function RollbackForeignTransaction;
+	PrepareForeignTransaction_function PrepareForeignTransaction;
 } FdwRoutine;
 
 
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index d9e417bcd7..3bea64a44b 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -403,6 +403,7 @@ typedef enum NodeTag
 	T_AlterFdwStmt,
 	T_CreateForeignServerStmt,
 	T_AlterForeignServerStmt,
+	T_DropForeignServerStmt,
 	T_CreateUserMappingStmt,
 	T_AlterUserMappingStmt,
 	T_DropUserMappingStmt,
diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h
index def9651b34..4ecd453689 100644
--- a/src/include/nodes/parsenodes.h
+++ b/src/include/nodes/parsenodes.h
@@ -2387,7 +2387,7 @@ typedef struct AlterFdwStmt
 } AlterFdwStmt;
 
 /* ----------------------
- *		Create/Alter FOREIGN SERVER Statements
+ *		Create/Alter/Drop FOREIGN SERVER Statements
  * ----------------------
  */
 
@@ -2411,6 +2411,14 @@ typedef struct AlterForeignServerStmt
 	bool		has_version;	/* version specified */
 } AlterForeignServerStmt;
 
+typedef struct DropForeignServerStmt
+{
+	NodeTag		type;
+	char	   *servername;		/* server name */
+	bool		missing_ok;		/* Skip error if missing? */
+	DropBehavior behavior;		/* RESTRICT or CASCADE behavior */
+} DropForeignServerStmt;
+
 /* ----------------------
  *		Create FOREIGN TABLE Statement
  * ----------------------
diff --git a/src/include/replication/syncrep.h b/src/include/replication/syncrep.h
index 4266afde8b..476fe8c688 100644
--- a/src/include/replication/syncrep.h
+++ b/src/include/replication/syncrep.h
@@ -82,7 +82,7 @@ extern char *syncrep_parse_error_msg;
 extern char *SyncRepStandbyNames;
 
 /* called by user backend */
-extern void SyncRepWaitForLSN(XLogRecPtr lsn, bool commit);
+extern bool SyncRepWaitForLSN(XLogRecPtr lsn, bool commit);
 
 /* called at backend exit */
 extern void SyncRepCleanupAtProcExit(void);
diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h
index b01fa52139..300a4cf5b6 100644
--- a/src/include/storage/procarray.h
+++ b/src/include/storage/procarray.h
@@ -93,5 +93,6 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin,
 
 extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin,
 											TransactionId *catalog_xmin);
+extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin);
 
 #endif							/* PROCARRAY_H */
diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h
index 6b40f1eeb8..35802eac86 100644
--- a/src/include/utils/guc_tables.h
+++ b/src/include/utils/guc_tables.h
@@ -89,6 +89,8 @@ enum config_group
 	CLIENT_CONN_PRELOAD,
 	CLIENT_CONN_OTHER,
 	LOCK_MANAGEMENT,
+	FOREIGN_TRANSACTION,
+	FOREIGN_TRANSACTION_RESOLVER,
 	COMPAT_OPTIONS_PREVIOUS,
 	COMPAT_OPTIONS_CLIENT,
 	ERROR_HANDLING_OPTIONS,
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index 6c6ec2e711..915c174d1c 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -224,7 +224,12 @@ typedef enum
 	WAIT_EVENT_LOGICAL_CHANGES_READ,
 	WAIT_EVENT_LOGICAL_CHANGES_WRITE,
 	WAIT_EVENT_LOGICAL_SUBXACT_READ,
-	WAIT_EVENT_LOGICAL_SUBXACT_WRITE
+	WAIT_EVENT_LOGICAL_SUBXACT_WRITE,
+	WAIT_EVENT_FDWXACT_FILE_READ,
+	WAIT_EVENT_FDWXACT_FILE_SYNC,
+	WAIT_EVENT_FDWXACT_FILE_WRITE,
+	WAIT_EVENT_FDWXACT_LAUNCHER_MAIN,
+	WAIT_EVENT_FDWXACT_RESOLVER_MAIN
 } WaitEventIO;
 
 
diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out
index e5ab11275d..443a6715df 100644
--- a/src/test/regress/expected/rules.out
+++ b/src/test/regress/expected/rules.out
@@ -1352,6 +1352,14 @@ pg_file_settings| SELECT a.sourcefile,
     a.applied,
     a.error
    FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error);
+pg_foreign_xacts| SELECT f.xid,
+    f.umid,
+    f.ownerid,
+    f.dbid,
+    f.state,
+    f.identifier,
+    f.locker_pid
+   FROM pg_foreign_xacts() f(xid, umid, ownerid, dbid, state, identifier, locker_pid);
 pg_group| SELECT pg_authid.rolname AS groname,
     pg_authid.oid AS grosysid,
     ARRAY( SELECT pg_auth_members.member
-- 
2.24.3 (Apple Git-128)

