[PATCH] Reuse Workers and Replication Slots during Logical Replication

From a4e9a10b695155111bcb32058dc35ef7f1aeb22f Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH] Reuse Logical Replication Background worker

This commit allows tablesync workers to move to another table that needs synchronization,
when they're done with the curren table in tablesync phase of Logical
Replication.

It reduces the overhead of launching/killing a new background worker for
each table. By reusing tablesync workers, replication slots created for
tablesync can be reused as well. Removing the burden of
creating/dropping replication slot improves tablesync speed.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                  |   9 +
 src/backend/catalog/pg_subscription.c       | 241 +++++++++++++-
 src/backend/commands/subscriptioncmds.c     | 168 ++++++----
 src/backend/replication/logical/launcher.c  |   4 +
 src/backend/replication/logical/tablesync.c | 202 +++++++++---
 src/backend/replication/logical/worker.c    | 345 ++++++++++++--------
 src/include/catalog/pg_subscription_rel.h   |  11 +-
 src/include/replication/slot.h              |   3 +-
 src/include/replication/worker_internal.h   |  20 ++
 src/test/regress/expected/misc_sanity.out   |   3 +-
 10 files changed, 766 insertions(+), 240 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index a186e35f00..39b5da4710 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8041,6 +8041,15 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>text</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation 
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index 33ae3da8ae..e39cee30fe 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -284,7 +284,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -313,9 +313,17 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+	{
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	}
+	else
+	{
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	}
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
-
+ 
 	/* Insert tuple into catalog. */
 	CatalogTupleInsert(rel, tup);
 
@@ -325,6 +333,42 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+						   			XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -355,14 +399,94 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name and state of a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRelStateAndSlot(Oid subid, 
+								Oid relid, 
+								char state,
+						   		XLogRecPtr sublsn,
+								char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -374,6 +498,46 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get replication slot name of subscription table.
+ *
+ * Returns null if the subscription table does not have a replication slot.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = TextDatumGetCString(d);
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
@@ -643,3 +807,62 @@ GetSubscriptionNotReadyRelations(Oid subid)
 
 	return res;
 }
+
+/*
+ * Get all relations for subscription that are in init state.
+ *
+ * Returned list is palloc'ed in current memory context.
+ */
+List *
+GetSubscriptionInitStateRelations(Oid subid)
+{
+	List	   *res = NIL;
+	Relation	rel;
+	HeapTuple	tup;
+	int			nkeys = 0;
+	ScanKeyData skey[2];
+	SysScanDesc scan;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	ScanKeyInit(&skey[nkeys++],
+				Anum_pg_subscription_rel_srsubid,
+				BTEqualStrategyNumber, F_OIDEQ,
+				ObjectIdGetDatum(subid));
+
+	ScanKeyInit(&skey[nkeys++],
+				Anum_pg_subscription_rel_srsubstate,
+				BTEqualStrategyNumber, F_CHAREQ,
+				CharGetDatum(SUBREL_STATE_INIT));
+
+	scan = systable_beginscan(rel, InvalidOid, false,
+							  NULL, nkeys, skey);
+
+	while (HeapTupleIsValid(tup = systable_getnext(scan)))
+	{
+		Form_pg_subscription_rel subrel;
+		SubscriptionRelState *relstate;
+		Datum		d;
+		bool		isnull;
+
+		subrel = (Form_pg_subscription_rel) GETSTRUCT(tup);
+
+		relstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+		relstate->relid = subrel->srrelid;
+		relstate->state = subrel->srsubstate;
+		d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+							Anum_pg_subscription_rel_srsublsn, &isnull);
+		if (isnull)
+			relstate->lsn = InvalidXLogRecPtr;
+		else
+			relstate->lsn = DatumGetLSN(d);
+
+		res = lappend(res, relstate);
+	}
+
+	/* Cleanup */
+	systable_endscan(scan);
+	table_close(rel, AccessShareLock);
+
+	return res;
+}
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index bd0cc0848d..fd1cea16b9 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -704,7 +704,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL);
 			}
 
 			/*
@@ -794,6 +794,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -866,7 +868,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -916,7 +918,18 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/* 
+				 * Find the logical replication sync worker if exists 
+				 * Store the slot number for dropping associated replication slot later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -950,31 +963,23 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
+		 * Drop the replication slots associated with tablesync workers for removed tables.
+		 * This has to be at the end because otherwise if there is an error while doing
 		 * the database operations we won't be able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			/*
+			 * There is no certainty, maybe the slot
+			 * does not exist yet. Also, if we fail after removing some of
+			 * the slots, next time, it will again try to drop already
+			 * dropped slots and fail. For these reasons, we allow
+			 * missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1567,39 +1572,16 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
-
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
-
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		List *slots = NULL;
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+		
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char *syncslotname = (char *) lfirst(lc);
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1622,6 +1604,69 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * WRITE COMMENT HERE
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok){
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+						" FROM pg_replication_slots"
+						" WHERE slot_name LIKE 'pg_%i_sync_%%';",
+						 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					 errmsg("not tuple returned."));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();\
+	return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -1863,6 +1908,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -1873,18 +1919,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+		
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 3bbd522724..b673bc9d27 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -370,7 +370,10 @@ retry:
 	/* Prepare the worker slot. */
 	worker->launch_time = now;
 	worker->in_use = true;
+	worker->is_first_run = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -378,6 +381,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->move_to_next_rel = false;
 	worker->stream_fileset = NULL;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 670c6fcada..b62dcf661e 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -126,12 +126,8 @@ static bool FetchTableStates(bool *started_tx);
 
 static StringInfo copybuf = NULL;
 
-/*
- * Exit routine for synchronization worker.
- */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +139,27 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/* Disconnect from publisher.
+	 * Otherwise reused sync workers causes exceeding max_wal_senders 
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the main apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+	
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -180,7 +185,7 @@ wait_for_relation_state_change(Oid relid, char expected_state)
 		LogicalRepWorker *worker;
 		XLogRecPtr	statelsn;
 
-		CHECK_FOR_INTERRUPTS();
+		CHECK_FOR_INTERRUPTS();		
 
 		InvalidateCatalogSnapshot();
 		state = GetSubscriptionRelState(MyLogicalRepWorker->subid,
@@ -284,6 +289,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -299,7 +308,6 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 
 		/*
 		 * UpdateSubscriptionRelState must be called within a transaction.
-		 * That transaction will be ended within the finish_sync_worker().
 		 */
 		if (!IsTransactionState())
 			StartTransactionCommand();
@@ -308,6 +316,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
 		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
@@ -317,24 +326,86 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 
 		/*
 		 * Cleanup the tablesync slot.
+		 * If the slot name used by this worker is different from the default 
+		 * slot name for the worker, this means the current table had started to being 
+		 * synchronized by another worker and replication slot. And this worker
+		 * is reusing a replication slot from a previous attempt.
+		 * We do not need that replication slot anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
 										syncslotname,
 										sizeof(syncslotname));
 
+
+		/* This transaction will be ended within the clean_sync_worker(). */
+		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+			UpdateSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+												 MyLogicalRepWorker->relid,
+												 NULL);
+		}
+
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
+		 * Check if any table whose relation state is still INIT. 
+		 * If a table in INIT state is found, the worker will not be finished,
+		 * it will be reused instead.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+		rstates = GetSubscriptionInitStateRelations(MySubscription->oid);
+		
+		foreach (lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
 
-		finish_sync_worker();
+			/* 
+			 * Pick the table for the next run
+			 * if there is not another worker already picked that table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				ereport(LOG,
+						(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+								MySubscription->name,
+								get_rel_name(MyLogicalRepWorker->relid))));
+
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->is_first_run = false;
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->move_to_next_rel = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* Cleanup before next run or ending the worker. */
+		if(!MyLogicalRepWorker->move_to_next_rel)
+		{
+		   /*
+			* It is important to give an error if we are unable to drop the slot,
+			* otherwise, it won't be dropped till the corresponding subscription
+			* is dropped. So passing missing_ok = false.
+			*/
+			if (MyLogicalRepWorker->created_slot)
+			{
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
+
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -473,9 +544,9 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
+				UpdateSubscriptionRelStateAndSlot(MyLogicalRepWorker->subid,
 										   rstate->relid, rstate->state,
-										   rstate->lsn);
+										   rstate->lsn, NULL);
 			}
 		}
 		else
@@ -1140,8 +1211,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%i_UINT64_FORMAT (3 + 10 + 6 + 5 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1152,11 +1223,10 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
-								char *syncslotname, int szslot)
+ReplicationSlotNameForTablesync(Oid suboid, char *syncslotname, int szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%i_" UINT64_FORMAT, suboid,
+			 MyProcPid, GetSystemIdentifier());
 }
 
 /*
@@ -1191,6 +1261,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char  		*prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1219,7 +1290,6 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
 									slotname,
 									NAMEDATALEN);
 
@@ -1245,6 +1315,20 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 									  originname,
 									  sizeof(originname));
 
+
+	/*
+	 * See if tablesync of the current relation has been 
+	 * started with another replication slot. 
+	 * 
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc0(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1258,10 +1342,34 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strcmp(prev_slotname, ""))
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+		
+			StartTransactionCommand();
+			UpdateSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+												  MyLogicalRepWorker->relid,
+												  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized should have
+		 * its replication slot name filled in the catalog. The tablesync process was started 
+		 * with another sync worker and replication slot.
+		 * We need to continue using the same replication slot in this worker too.
+		 */
+		if (!strcmp(prev_slotname, ""))
+		{
+			elog(ERROR, "Replication slot could not be found for relation %u",
+				 MyLogicalRepWorker->relid);
+		}
+
+		/* Proceed with the correct replication slot. */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1289,10 +1397,11 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
+	UpdateSubscriptionRelStateAndSlot(MyLogicalRepWorker->subid,
 							   MyLogicalRepWorker->relid,
 							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+							   MyLogicalRepWorker->relstate_lsn,
+							   slotname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1351,16 +1460,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
 	 *
+	 * Replication slot will only be created if either this is the first run 
+	 * of the worker or we're not using a previous replication slot. 
+	 * 
 	 * Prevent cancel/die interrupts while creating slot here because it is
 	 * possible that before the server finishes this command, a concurrent
 	 * drop subscription happens which would complete without removing this
 	 * slot leading to a dangling slot on the server.
 	 */
-	HOLD_INTERRUPTS();
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-	RESUME_INTERRUPTS();
+	if (MyLogicalRepWorker->is_first_run ||
+		(!strcmp(prev_slotname, "") && !MyLogicalRepWorker->created_slot))
+	{
+		HOLD_INTERRUPTS();
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						slotname, false /* permanent */ , false /* two_phase */ ,
+						CRS_USE_SNAPSHOT, origin_startpos);
+		RESUME_INTERRUPTS();
+
+		/*
+		 * Remember that we created the slot so that
+		 * we will not try to create it again.
+		 */
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
+	}
 
 	/*
 	 * Setup replication origin tracking. The purpose of doing this before the
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 5f8c541763..163c0ea921 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -315,6 +315,7 @@ static void stream_cleanup_files(Oid subid, TransactionId xid);
 static void stream_open_file(Oid subid, TransactionId xid, bool first);
 static void stream_write_change(char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -2814,6 +2815,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->move_to_next_rel)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -2915,8 +2920,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	/* Pop the error context stack */
 	error_context_stack = errcallback.previous;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/* 
+	 * If it's moving to next relation, this is a sync worker.
+	 * Sync workers end the streaming during process_syncing_tables_for_sync.
+	 * Calling endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->move_to_next_rel)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -3458,6 +3471,35 @@ stream_write_change(char action, StringInfo s)
 	BufFileWrite(stream_fd, &s->data[s->cursor], len);
 }
 
+/*
+ * stream_build_options_replication
+ * 		Build logical replication streaming options.
+ *
+ * This function sets streaming options including replication slot name
+ * and origin start position. Workers need these options for logical replication.
+ */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.streaming = MySubscription->stream;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -3532,6 +3574,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -3569,6 +3614,136 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync, 
+ * builds streaming options and starts streaming. 
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options, 
+					 char *slotname,
+					 char *originname,
+					 int originame_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+    MyLogicalRepWorker->move_to_next_rel = false;
+
+    start_table_sync(origin_startpos, &slotname);
+
+    /*
+        * Allocate the origin name in long-lived context for error context
+        * message.
+        */
+    ReplicationOriginNameForTablesync(MySubscription->oid,
+                                        MyLogicalRepWorker->relid,
+                                        originname,
+                                        originame_size);
+    apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+                                                                originname);
+    
+    stream_build_options(options, slotname, origin_startpos);
+
+    /* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options 
+ * and then starts streaming. 
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+    RepOriginId originid;
+    TimeLineID	startpointTLI;
+    char	   *err;
+
+    slotname = MySubscription->slotname;
+
+    /*
+	 * This shouldn't happen if the subscription is enabled, but guard
+	 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+	 * crash if slot is NULL.)
+	 */
+    if (!slotname)
+        ereport(ERROR,
+                (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+                    errmsg("subscription has no replication slot set")));
+
+    /* Setup replication origin tracking. */
+    StartTransactionCommand();
+    snprintf(originname, originname_size, "pg_%u", MySubscription->oid);
+    originid = replorigin_by_name(originname, true);
+    if (!OidIsValid(originid))
+        originid = replorigin_create(originname);
+    replorigin_session_setup(originid);
+    replorigin_session_origin = originid;
+    *origin_startpos = replorigin_session_get_progress(false);
+    CommitTransactionCommand();
+
+    LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+                                            MySubscription->name, &err);
+    if (LogRepWorkerWalRcvConn == NULL)
+        ereport(ERROR,
+                (errcode(ERRCODE_CONNECTION_FAILURE),
+                    errmsg("could not connect to the publisher: %s", err)));
+
+    /*
+	 * We don't really use the output identify_system for anything but it
+	 * does some initializations on the upstream so let's still call it.
+	 */
+    (void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+    /*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+    apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+                                                                originname);
+
+    stream_build_options(options, slotname, origin_startpos);
+
+    /*
+     * Even when the two_phase mode is requested by the user, it remains
+     * as the tri-state PENDING until all tablesyncs have reached READY
+     * state. Only then, can it become ENABLED.
+     *
+     * Note: If the subscription has no tables then leave the state as
+     * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+     * work.
+     */
+    if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+        AllTablesyncsReady())
+    {
+        /* Start streaming with two_phase enabled */
+        options->proto.logical.twophase = true;
+        walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+        StartTransactionCommand();
+        UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+        MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+        CommitTransactionCommand();
+    }
+    else
+    {
+        walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+    }
+
+    ereport(DEBUG1,
+            (errmsg("logical replication apply worker for subscription \"%s\" two_phase is %s",
+                    MySubscription->name,
+                    MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+                    MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+                    MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+                    "?")));
+}
+
 /* Logical Replication Apply worker entry point */
 void
 ApplyWorkerMain(Datum main_arg)
@@ -3579,7 +3754,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -3670,142 +3844,55 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		/*
-		 * Allocate the origin name in long-lived context for error context
-		 * message.
-		 */
-		ReplicationOriginNameForTablesync(MySubscription->oid,
-										  MyLogicalRepWorker->relid,
-										  originname,
-										  sizeof(originname));
-		apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
-																   originname);
-	}
-	else
-	{
-		/* This is main apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		snprintf(originname, sizeof(originname), "pg_%u", MySubscription->oid);
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		/*
-		 * Allocate the origin name in long-lived context for error context
-		 * message.
-		 */
-		apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
-																   originname);
-	}
-
 	/*
-	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
-	 */
+	* Setup callback for syscache so that we know when something changes in
+	* the subscription relation state.
+	* Do this outside the loop to avoid exceeding MAX_SYSCACHE_CALLBACKS
+	*/
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
-								  invalidate_syncing_table_states,
-								  (Datum) 0);
-
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-	options.proto.logical.streaming = MySubscription->stream;
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
+								invalidate_syncing_table_states,
+								(Datum) 0);
 
-	if (!am_tablesync_worker())
+	/*
+	 * The loop where worker does its job.
+	 * It loops until the worker is not reused. 
+	 */
+	while (MyLogicalRepWorker->is_first_run || 
+			MyLogicalRepWorker->move_to_next_rel)
 	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		if (am_tablesync_worker())
+			{
+				/* 
+				* This is a tablesync worker. 
+				* Start syncing tables before starting the apply loop.  
+				*/
+				run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+			}
+			else
+			{
+				/* This is main apply worker */
+				run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+			}
+		
+		/* Run the main loop. */
+		start_apply(origin_startpos);
+
+		if (MyLogicalRepWorker->move_to_next_rel)
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* Reset the currenct replication origin session.
+			* Since we'll use the same process for another relation, it needs to be reset 
+			* and will be created again later while syncing the new relation.
+			*/
+			replorigin_session_origin = InvalidRepOriginId;
+			replorigin_session_reset();
 
 			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
 			CommitTransactionCommand();
 		}
-		else
-		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-		}
-
-		ereport(DEBUG1,
-				(errmsg("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
 	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
-
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 9df99c3418..b127817b66 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,8 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	text		srrelslotname BKI_FORCE_NULL;	/* name of the replication slot for relatio in subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,14 +83,21 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+
+extern void UpdateSubscriptionRelStateAndSlot(Oid subid, Oid relid, char state,
+									XLogRecPtr sublsn, char *relslotname);
+
 extern bool HasSubscriptionRelations(Oid subid);
 extern List *GetSubscriptionRelations(Oid subid);
 extern List *GetSubscriptionNotReadyRelations(Oid subid);
+extern List *GetSubscriptionInitStateRelations(Oid subid);
 
 #endif							/* PG_SUBSCRIPTION_REL_H */
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8c9f3321d5..6bdf1d1238 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -217,8 +217,9 @@ extern void ReplicationSlotsDropDBSlots(Oid dboid);
 extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, int szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, char *syncslotname, int szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List* GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 901845abc2..2180ffca11 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -30,6 +30,21 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/* 
+	 * Indicates if worker is running for the first time
+	 * or in reuse
+	 */
+	bool		is_first_run;
+
+	/* 
+	 * Indicates if the sync worker created a replication slot
+	 * or it reuses an existing one created by another worker.
+	 */
+	bool		created_slot;
+
+	/* Replication slot name used by the worker. */
+	char		*slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -51,6 +66,11 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/* 
+	 * Used to indicate whether sync worker will be reused for another relation
+	 */
+	bool		move_to_next_rel;
+	
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction, the
diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out
index a57fd142a9..a2a6f14944 100644
--- a/src/test/regress/expected/misc_sanity.out
+++ b/src/test/regress/expected/misc_sanity.out
@@ -60,7 +60,8 @@ ORDER BY 1, 2;
  pg_index                | indpred       | pg_node_tree
  pg_largeobject          | data          | bytea
  pg_largeobject_metadata | lomacl        | aclitem[]
-(11 rows)
+ pg_subscription_rel     | srrelslotname | text
+(12 rows)
 
 -- system catalogs without primary keys
 --
-- 
2.25.1

From e3d9ab01146c78f8c8e2954c11c0f0d72bf204f4 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH] Reuse Logical Replication Background worker

This commit allows tablesync workers to move to another table that needs synchronization,
when they're done with the curren table in tablesync phase of Logical Replication.

It reduces the overhead of launching/killing a new background worker for each table.
By reusing tablesync workers, replication slots created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot improves tablesync speed.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                  |  20 ++
 src/backend/catalog/pg_subscription.c       | 231 ++++++++++++-
 src/backend/commands/subscriptioncmds.c     | 169 ++++++----
 src/backend/replication/logical/launcher.c  |   9 +-
 src/backend/replication/logical/tablesync.c | 217 +++++++++---
 src/backend/replication/logical/worker.c    | 345 ++++++++++++--------
 src/include/catalog/pg_subscription.h       |   5 +
 src/include/catalog/pg_subscription_rel.h   |  10 +-
 src/include/replication/slot.h              |   3 +-
 src/include/replication/worker_internal.h   |  28 +-
 src/test/regress/expected/misc_sanity.out   |   3 +-
 11 files changed, 794 insertions(+), 246 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index cd2cc37aeb..8048710244 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7957,6 +7957,17 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+       The last used ID for tablesync workers. This ID is used to 
+       create replication slots. The last used ID needs to be stored 
+       to make logical replication can safely proceed after an interruption.  
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8041,6 +8052,15 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>text</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation 
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a506fc3ec8..fc7973859d 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -71,7 +71,6 @@ GetSubscription(Oid subid, bool missing_ok)
 	sub->stream = subform->substream;
 	sub->twophasestate = subform->subtwophasestate;
 	sub->disableonerr = subform->subdisableonerr;
-
 	/* Get conninfo */
 	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
 							tup,
@@ -114,6 +113,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +212,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +279,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,9 +308,17 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+	{
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	}
+	else
+	{
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	}
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
-
+ 
 	/* Insert tuple into catalog. */
 	CatalogTupleInsert(rel, tup);
 
@@ -275,6 +328,42 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+						   			XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +394,94 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name and state of a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRelStateAndSlot(Oid subid, 
+								Oid relid, 
+								char state,
+						   		XLogRecPtr sublsn,
+								char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +493,46 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get replication slot name of subscription table.
+ *
+ * Returns null if the subscription table does not have a replication slot.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = TextDatumGetCString(d);
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
@@ -542,4 +751,4 @@ GetSubscriptionRelations(Oid subid, bool not_ready)
 	table_close(rel, AccessShareLock);
 
 	return res;
-}
+}
\ No newline at end of file
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index f73dfb6067..b893700a84 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -646,6 +646,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(1);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -704,7 +705,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL);
 			}
 
 			/*
@@ -794,6 +795,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -866,7 +869,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -916,7 +919,18 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/* 
+				 * Find the logical replication sync worker if exists 
+				 * Store the slot number for dropping associated replication slot later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -950,31 +964,23 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
+		 * Drop the replication slots associated with tablesync workers for removed tables.
+		 * This has to be at the end because otherwise if there is an error while doing
 		 * the database operations we won't be able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			/*
+			 * There is no certainty, maybe the slot
+			 * does not exist yet. Also, if we fail after removing some of
+			 * the slots, next time, it will again try to drop already
+			 * dropped slots and fail. For these reasons, we allow
+			 * missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1567,39 +1573,16 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
-
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
-
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		List *slots = NULL;
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+		
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char *syncslotname = (char *) lfirst(lc);
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1622,6 +1605,69 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * WRITE COMMENT HERE
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok){
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+						" FROM pg_replication_slots"
+						" WHERE slot_name LIKE 'pg_%i_sync_%%';",
+						 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					 errmsg("not tuple returned."));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();\
+	return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -1863,6 +1909,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -1873,18 +1920,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+		
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 3bbd522724..96cc38eeb5 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -263,7 +263,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 void
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid)
+						 Oid relid, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -370,7 +370,11 @@ retry:
 	/* Prepare the worker slot. */
 	worker->launch_time = now;
 	worker->in_use = true;
+	worker->is_first_run = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -378,6 +382,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->move_to_next_rel = false;
 	worker->stream_fileset = NULL;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
@@ -868,7 +873,7 @@ ApplyLauncherMain(Datum main_arg)
 					wait_time = wal_retrieve_retry_interval;
 
 					logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
-											 sub->owner, InvalidOid);
+											 sub->owner, InvalidOid, 0);
 				}
 			}
 
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 6a01ffd273..a561e6f29d 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -126,12 +126,8 @@ static bool FetchTableStates(bool *started_tx);
 
 static StringInfo copybuf = NULL;
 
-/*
- * Exit routine for synchronization worker.
- */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +139,27 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/* Disconnect from publisher.
+	 * Otherwise reused sync workers causes exceeding max_wal_senders 
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the main apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+	
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -180,7 +185,7 @@ wait_for_relation_state_change(Oid relid, char expected_state)
 		LogicalRepWorker *worker;
 		XLogRecPtr	statelsn;
 
-		CHECK_FOR_INTERRUPTS();
+		CHECK_FOR_INTERRUPTS();		
 
 		InvalidateCatalogSnapshot();
 		state = GetSubscriptionRelState(MyLogicalRepWorker->subid,
@@ -284,6 +289,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -299,7 +308,6 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 
 		/*
 		 * UpdateSubscriptionRelState must be called within a transaction.
-		 * That transaction will be ended within the finish_sync_worker().
 		 */
 		if (!IsTransactionState())
 			StartTransactionCommand();
@@ -308,6 +316,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
 		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
@@ -317,24 +326,88 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 
 		/*
 		 * Cleanup the tablesync slot.
+		 * If the slot name used by this worker is different from the default 
+		 * slot name for the worker, this means the current table had started to being 
+		 * synchronized by another worker and replication slot. And this worker
+		 * is reusing a replication slot from a previous attempt.
+		 * We do not need that replication slot anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
+
+		/* This transaction will be ended within the clean_sync_worker(). */
+		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+			UpdateSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+												 MyLogicalRepWorker->relid,
+												 NULL);
+		}
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid))));
+
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
+		 * Check if any table whose relation state is still INIT. 
+		 * If a table in INIT state is found, the worker will not be finished,
+		 * it will be reused instead.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
+		
+		foreach (lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+			/* 
+			 * Pick the table for the next run
+			 * if there is not another worker already picked that table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE && 
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->is_first_run = false;
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->move_to_next_rel = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* Cleanup before next run or ending the worker. */
+		if(!MyLogicalRepWorker->move_to_next_rel)
+		{
+		   /*
+			* It is important to give an error if we are unable to drop the slot,
+			* otherwise, it won't be dropped till the corresponding subscription
+			* is dropped. So passing missing_ok = false.
+			*/
+			if (MyLogicalRepWorker->created_slot)
+			{
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
 
-		finish_sync_worker();
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -473,9 +546,9 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
+				UpdateSubscriptionRelStateAndSlot(MyLogicalRepWorker->subid,
 										   rstate->relid, rstate->state,
-										   rstate->lsn);
+										   rstate->lsn, NULL);
 			}
 		}
 		else
@@ -564,11 +637,21 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													MySubscription->lastusedid);
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
-												 rstate->relid);
+												 rstate->relid,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1140,8 +1223,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1152,11 +1235,10 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
-								char *syncslotname, int szslot)
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, int szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lu_" UINT64_FORMAT, suboid,
+			 slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1191,6 +1273,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char  		*prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1219,7 +1302,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1245,6 +1328,20 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 									  originname,
 									  sizeof(originname));
 
+
+	/*
+	 * See if tablesync of the current relation has been 
+	 * started with another replication slot. 
+	 * 
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc0(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1258,10 +1355,34 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strcmp(prev_slotname, ""))
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+		
+			StartTransactionCommand();
+			UpdateSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+												  MyLogicalRepWorker->relid,
+												  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized should have
+		 * its replication slot name filled in the catalog. The tablesync process was started 
+		 * with another sync worker and replication slot.
+		 * We need to continue using the same replication slot in this worker too.
+		 */
+		if (!strcmp(prev_slotname, ""))
+		{
+			elog(ERROR, "Replication slot could not be found for relation %u",
+				 MyLogicalRepWorker->relid);
+		}
+
+		/* Proceed with the correct replication slot. */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1289,10 +1410,11 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
+	UpdateSubscriptionRelStateAndSlot(MyLogicalRepWorker->subid,
 							   MyLogicalRepWorker->relid,
 							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+							   MyLogicalRepWorker->relstate_lsn,
+							   slotname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1351,16 +1473,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
 	 *
+	 * Replication slot will only be created if either this is the first run 
+	 * of the worker or we're not using a previous replication slot. 
+	 * 
 	 * Prevent cancel/die interrupts while creating slot here because it is
 	 * possible that before the server finishes this command, a concurrent
 	 * drop subscription happens which would complete without removing this
 	 * slot leading to a dangling slot on the server.
 	 */
-	HOLD_INTERRUPTS();
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-	RESUME_INTERRUPTS();
+	if (MyLogicalRepWorker->is_first_run ||
+		(!strcmp(prev_slotname, "") && !MyLogicalRepWorker->created_slot))
+	{
+		HOLD_INTERRUPTS();
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						slotname, false /* permanent */ , false /* two_phase */ ,
+						CRS_USE_SNAPSHOT, origin_startpos);
+		RESUME_INTERRUPTS();
+
+		/*
+		 * Remember that we created the slot so that
+		 * we will not try to create it again.
+		 */
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
+	}
 
 	/*
 	 * Setup replication origin tracking. The purpose of doing this before the
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 5f8c541763..163c0ea921 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -315,6 +315,7 @@ static void stream_cleanup_files(Oid subid, TransactionId xid);
 static void stream_open_file(Oid subid, TransactionId xid, bool first);
 static void stream_write_change(char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -2814,6 +2815,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->move_to_next_rel)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -2915,8 +2920,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	/* Pop the error context stack */
 	error_context_stack = errcallback.previous;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/* 
+	 * If it's moving to next relation, this is a sync worker.
+	 * Sync workers end the streaming during process_syncing_tables_for_sync.
+	 * Calling endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->move_to_next_rel)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -3458,6 +3471,35 @@ stream_write_change(char action, StringInfo s)
 	BufFileWrite(stream_fd, &s->data[s->cursor], len);
 }
 
+/*
+ * stream_build_options_replication
+ * 		Build logical replication streaming options.
+ *
+ * This function sets streaming options including replication slot name
+ * and origin start position. Workers need these options for logical replication.
+ */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.streaming = MySubscription->stream;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -3532,6 +3574,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -3569,6 +3614,136 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync, 
+ * builds streaming options and starts streaming. 
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options, 
+					 char *slotname,
+					 char *originname,
+					 int originame_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+    MyLogicalRepWorker->move_to_next_rel = false;
+
+    start_table_sync(origin_startpos, &slotname);
+
+    /*
+        * Allocate the origin name in long-lived context for error context
+        * message.
+        */
+    ReplicationOriginNameForTablesync(MySubscription->oid,
+                                        MyLogicalRepWorker->relid,
+                                        originname,
+                                        originame_size);
+    apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+                                                                originname);
+    
+    stream_build_options(options, slotname, origin_startpos);
+
+    /* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options 
+ * and then starts streaming. 
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+    RepOriginId originid;
+    TimeLineID	startpointTLI;
+    char	   *err;
+
+    slotname = MySubscription->slotname;
+
+    /*
+	 * This shouldn't happen if the subscription is enabled, but guard
+	 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+	 * crash if slot is NULL.)
+	 */
+    if (!slotname)
+        ereport(ERROR,
+                (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+                    errmsg("subscription has no replication slot set")));
+
+    /* Setup replication origin tracking. */
+    StartTransactionCommand();
+    snprintf(originname, originname_size, "pg_%u", MySubscription->oid);
+    originid = replorigin_by_name(originname, true);
+    if (!OidIsValid(originid))
+        originid = replorigin_create(originname);
+    replorigin_session_setup(originid);
+    replorigin_session_origin = originid;
+    *origin_startpos = replorigin_session_get_progress(false);
+    CommitTransactionCommand();
+
+    LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+                                            MySubscription->name, &err);
+    if (LogRepWorkerWalRcvConn == NULL)
+        ereport(ERROR,
+                (errcode(ERRCODE_CONNECTION_FAILURE),
+                    errmsg("could not connect to the publisher: %s", err)));
+
+    /*
+	 * We don't really use the output identify_system for anything but it
+	 * does some initializations on the upstream so let's still call it.
+	 */
+    (void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+    /*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+    apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+                                                                originname);
+
+    stream_build_options(options, slotname, origin_startpos);
+
+    /*
+     * Even when the two_phase mode is requested by the user, it remains
+     * as the tri-state PENDING until all tablesyncs have reached READY
+     * state. Only then, can it become ENABLED.
+     *
+     * Note: If the subscription has no tables then leave the state as
+     * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+     * work.
+     */
+    if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+        AllTablesyncsReady())
+    {
+        /* Start streaming with two_phase enabled */
+        options->proto.logical.twophase = true;
+        walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+        StartTransactionCommand();
+        UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+        MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+        CommitTransactionCommand();
+    }
+    else
+    {
+        walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+    }
+
+    ereport(DEBUG1,
+            (errmsg("logical replication apply worker for subscription \"%s\" two_phase is %s",
+                    MySubscription->name,
+                    MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+                    MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+                    MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+                    "?")));
+}
+
 /* Logical Replication Apply worker entry point */
 void
 ApplyWorkerMain(Datum main_arg)
@@ -3579,7 +3754,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -3670,142 +3844,55 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		/*
-		 * Allocate the origin name in long-lived context for error context
-		 * message.
-		 */
-		ReplicationOriginNameForTablesync(MySubscription->oid,
-										  MyLogicalRepWorker->relid,
-										  originname,
-										  sizeof(originname));
-		apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
-																   originname);
-	}
-	else
-	{
-		/* This is main apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		snprintf(originname, sizeof(originname), "pg_%u", MySubscription->oid);
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		/*
-		 * Allocate the origin name in long-lived context for error context
-		 * message.
-		 */
-		apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
-																   originname);
-	}
-
 	/*
-	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
-	 */
+	* Setup callback for syscache so that we know when something changes in
+	* the subscription relation state.
+	* Do this outside the loop to avoid exceeding MAX_SYSCACHE_CALLBACKS
+	*/
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
-								  invalidate_syncing_table_states,
-								  (Datum) 0);
-
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-	options.proto.logical.streaming = MySubscription->stream;
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
+								invalidate_syncing_table_states,
+								(Datum) 0);
 
-	if (!am_tablesync_worker())
+	/*
+	 * The loop where worker does its job.
+	 * It loops until the worker is not reused. 
+	 */
+	while (MyLogicalRepWorker->is_first_run || 
+			MyLogicalRepWorker->move_to_next_rel)
 	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		if (am_tablesync_worker())
+			{
+				/* 
+				* This is a tablesync worker. 
+				* Start syncing tables before starting the apply loop.  
+				*/
+				run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+			}
+			else
+			{
+				/* This is main apply worker */
+				run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+			}
+		
+		/* Run the main loop. */
+		start_apply(origin_startpos);
+
+		if (MyLogicalRepWorker->move_to_next_rel)
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* Reset the currenct replication origin session.
+			* Since we'll use the same process for another relation, it needs to be reset 
+			* and will be created again later while syncing the new relation.
+			*/
+			replorigin_session_origin = InvalidRepOriginId;
+			replorigin_session_reset();
 
 			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
 			CommitTransactionCommand();
 		}
-		else
-		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-		}
-
-		ereport(DEBUG1,
-				(errmsg("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
 	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
-
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 7b98714f30..2d391615e8 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -102,6 +102,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);	
 #endif
 } FormData_pg_subscription;
 
@@ -135,11 +138,13 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication slots in tablesync */
 } Subscription;
 
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 8e88de7b2b..7373df40ab 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,8 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	text		srrelslotname BKI_FORCE_NULL;	/* name of the replication slot for relatio in subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,12 +83,18 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+
+extern void UpdateSubscriptionRelStateAndSlot(Oid subid, Oid relid, char state,
+									XLogRecPtr sublsn, char *relslotname);
+
 extern bool HasSubscriptionRelations(Oid subid);
 extern List *GetSubscriptionRelations(Oid subid, bool not_ready);
 
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8c9f3321d5..4b3e2993e3 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -217,8 +217,9 @@ extern void ReplicationSlotsDropDBSlots(Oid dboid);
 extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, int szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, int szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List* GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 901845abc2..5fabd1a829 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -30,6 +30,27 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/* 
+	 * Indicates if worker is running for the first time
+	 * or in reuse
+	 */
+	bool		is_first_run;
+
+	/* 
+	 * Indicates if the sync worker created a replication slot
+	 * or it reuses an existing one created by another worker.
+	 */
+	bool		created_slot;
+
+	/* 
+	 * Unique identifier for replication slot to be 
+	 * created by tablesnync workers, if needed.
+	 */
+	int64 	 	rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char		*slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -51,6 +72,11 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/* 
+	 * Used to indicate whether sync worker will be reused for another relation
+	 */
+	bool		move_to_next_rel;
+	
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction, the
@@ -85,7 +111,7 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 												bool only_running);
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
-									 Oid userid, Oid relid);
+									 Oid userid, Oid relid, int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
 extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker);
diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out
index a57fd142a9..a2a6f14944 100644
--- a/src/test/regress/expected/misc_sanity.out
+++ b/src/test/regress/expected/misc_sanity.out
@@ -60,7 +60,8 @@ ORDER BY 1, 2;
  pg_index                | indpred       | pg_node_tree
  pg_largeobject          | data          | bytea
  pg_largeobject_metadata | lomacl        | aclitem[]
-(11 rows)
+ pg_subscription_rel     | srrelslotname | text
+(12 rows)
 
 -- system catalogs without primary keys
 --
-- 
2.25.1

From 4bf4d25c2d34063d0018825349b61de0a8d80bf7 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 1/2] Add replication protocol cmd to create a snapshot

Introduced REPLICATION_SLOT_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

REPLICATION_SLOT_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.
---
 doc/src/sgml/protocol.sgml                    | 32 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       | 69 ++++++++++++-
 src/backend/replication/logical/logical.c     | 33 ++++++-
 .../replication/logical/logicalfuncs.c        |  1 +
 src/backend/replication/repl_gram.y           | 18 +++-
 src/backend/replication/repl_scanner.l        |  2 +
 src/backend/replication/slotfuncs.c           |  1 +
 src/backend/replication/walsender.c           | 97 ++++++++++++++++++-
 src/include/nodes/replnodes.h                 | 11 +++
 src/include/replication/logical.h             |  1 +
 src/include/replication/walreceiver.h         | 13 +++
 src/tools/pgindent/typedefs.list              |  1 +
 12 files changed, 275 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 5fdd429e05..e148e3baf0 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2612,6 +2612,38 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>REPLICATION_SLOT_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>REPLICATION_SLOT_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the currenct transaction. This command is currently only supported
+       for logical replication.
+       slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index 7f697b0f29..e367c5f757 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -80,6 +80,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -96,7 +98,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -968,6 +971,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "REPLICATION_SLOT_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 625a7f4273..56d2256c26 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -461,6 +461,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, create a snapshot able to read all tables,
+ * 		otherwise do not create any snapshot.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -479,6 +483,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -487,6 +492,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -533,8 +539,33 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 5c23178570..e4b80f2764 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 7ba33d6672..f70b2e087a 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_REPLICATION_SLOT_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show replication_slot_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| replication_slot_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * REPLICATION_SLOT_SNAPSHOT %s options
+ */
+replication_slot_snapshot:
+			K_REPLICATION_SLOT_SNAPSHOT var_name plugin_options
+				{
+					ReplicationSlotSnapshotCmd *n = makeNode(ReplicationSlotSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_REPLICATION_SLOT_SNAPSHOT	{ $$ = "replication_slot_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index 72ef12225e..e33030da61 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+REPLICATION_SLOT_SNAPSHOT	{ return K_REPLICATION_SLOT_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_REPLICATION_SLOT_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 79d7f1372d..13d099fe06 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -478,6 +478,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c11bb3716f..f9a22c8b16 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void ReplicationSlotSnapshot(ReplicationSlotSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1280,7 +1281,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1332,6 +1333,91 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+ReplicationSlotSnapshot(ReplicationSlotSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the replication slot and don't yet accept feedback messages or send
+	 * keepalives. As we possibly need to wait for further WAL the walsender
+	 * would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consisten WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1860,6 +1946,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_ReplicationSlotSnapshotCmd:
+			{
+				cmdtag = "REPLICATION_SLOT_SNAPSHOT";
+				set_ps_display(cmdtag);
+				ReplicationSlotSnapshot((ReplicationSlotSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 8ae9c517ff..94aa7d2253 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		REPLICATION_SLOT_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct ReplicationSlotSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} ReplicationSlotSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 4ad019e25a..3875b32f9d 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 9339f29303..274a162b48 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -384,6 +384,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -401,6 +411,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -435,6 +446,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 58daeca831..cc21aae582 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2313,6 +2313,7 @@ ReplicationSlotCtlData
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+ReplicationSlotSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
-- 
2.25.1

From 340f7aa726ccab46f5ea14b2ab406235fd1ef521 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 2/2] Reuse Logical Replication Background worker

This commit allows tablesync workers to move to another table that needs synchronization,
when they're done with the current table in tablesync phase of Logical Replication.

It reduces the overhead of launching/killing a new background worker for each table.
By reusing tablesync workers, replication slots created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot improves tablesync speed.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                  |  29 ++
 src/backend/catalog/pg_subscription.c       | 285 ++++++++++++-
 src/backend/commands/subscriptioncmds.c     | 238 +++++++----
 src/backend/replication/logical/launcher.c  |   9 +-
 src/backend/replication/logical/tablesync.c | 434 +++++++++++++++-----
 src/backend/replication/logical/worker.c    | 370 +++++++++++------
 src/include/catalog/pg_subscription.h       |   6 +
 src/include/catalog/pg_subscription_rel.h   |  15 +-
 src/include/replication/slot.h              |   3 +-
 src/include/replication/worker_internal.h   |  32 +-
 src/test/regress/expected/misc_sanity.out   |  30 +-
 11 files changed, 1095 insertions(+), 356 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 9ed2b020b7..e81a60c90e 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7987,6 +7987,17 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+       The last used ID for tablesync workers. This ID is used to
+       create replication slots. The last used ID needs to be stored
+       to make logical replication can safely proceed after an interruption.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8071,6 +8082,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>text</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>text</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a506fc3ec8..ef0c1ec48a 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -71,7 +71,6 @@ GetSubscription(Oid subid, bool missing_ok)
 	sub->stream = subform->substream;
 	sub->twophasestate = subform->subtwophasestate;
 	sub->disableonerr = subform->subdisableonerr;
-
 	/* Get conninfo */
 	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
 							tup,
@@ -114,6 +113,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +212,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +279,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,6 +308,14 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -275,6 +328,58 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +410,48 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +463,134 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Get origin name of subscription table.
+ *
+ * Returns null if the subscription table does not have a origin.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = TextDatumGetCString(d);
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * Returns null if the subscription table does not have a replication slot.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = TextDatumGetCString(d);
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index d673557ea4..1b861f129c 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -648,6 +648,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(1);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -657,7 +658,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 
 	recordDependencyOnOwner(SubscriptionRelationId, subid, owner);
 
-	ReplicationOriginNameForLogicalRep(subid, InvalidOid, originname, sizeof(originname));
+	ReplicationOriginNameForLogicalRep(subid, originname, sizeof(originname), false);
 	replorigin_create(originname);
 
 	/*
@@ -708,7 +709,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -798,6 +799,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -875,7 +878,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -899,6 +902,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -925,7 +929,29 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker if exists Store
+				 * the slot number for dropping associated replication slot
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -945,8 +971,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 					 * origin and by this time the origin might be already
 					 * removed. For these reasons, passing missing_ok = true.
 					 */
-					ReplicationOriginNameForLogicalRep(sub->oid, relid, originname,
-													   sizeof(originname));
+					ReplicationOriginNameForLogicalRep(sub->oid, originname,
+													   sizeof(originname), true);
 					replorigin_drop_by_name(originname, true, false);
 				}
 
@@ -959,31 +985,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1319,8 +1338,8 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt,
 					char		originname[NAMEDATALEN];
 					XLogRecPtr	remote_lsn;
 
-					ReplicationOriginNameForLogicalRep(subid, InvalidOid,
-													   originname, sizeof(originname));
+					ReplicationOriginNameForLogicalRep(subid, originname,
+													   sizeof(originname), false);
 					originid = replorigin_by_name(originname, false);
 					remote_lsn = replorigin_get_progress(originid, false);
 
@@ -1380,6 +1399,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1451,6 +1471,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1500,37 +1528,29 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * TODO - should handle better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%ld", subid, i);
+		/* missin_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1538,7 +1558,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	RemoveSubscriptionRel(subid, InvalidOid);
 
 	/* Remove the origin tracking if exists. */
-	ReplicationOriginNameForLogicalRep(subid, InvalidOid, originname, sizeof(originname));
+	ReplicationOriginNameForLogicalRep(subid, originname, sizeof(originname), false);
 	replorigin_drop_by_name(originname, true, false);
 
 	/*
@@ -1582,39 +1602,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1637,6 +1635,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * WRITE COMMENT HERE
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("not tuple returned."));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+	\
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -1989,6 +2052,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -1999,18 +2063,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 73594c698a..5b5dead996 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -264,7 +264,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 void
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid)
+						 Oid relid, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -371,7 +371,11 @@ retry:
 	/* Prepare the worker slot. */
 	worker->launch_time = now;
 	worker->in_use = true;
+	worker->is_first_run = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -379,6 +383,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->move_to_next_rel = false;
 	worker->stream_fileset = NULL;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
@@ -869,7 +874,7 @@ ApplyLauncherMain(Datum main_arg)
 					wait_time = wal_retrieve_retry_interval;
 
 					logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
-											 sub->owner, InvalidOid);
+											 sub->owner, InvalidOid, 0);
 				}
 			}
 
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 94e813ac53..a6364d9433 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -126,12 +126,8 @@ static bool FetchTableStates(bool *started_tx);
 
 static StringInfo copybuf = NULL;
 
-/*
- * Exit routine for synchronization worker.
- */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +139,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the main apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -284,6 +290,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -292,6 +302,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		TimeLineID	tli;
 		char		syncslotname[NAMEDATALEN] = {0};
 		char		originname[NAMEDATALEN] = {0};
+		bool		is_streaming_ended = false;
 
 		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
 		MyLogicalRepWorker->relstate_lsn = current_lsn;
@@ -308,40 +319,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication trackin origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -350,34 +350,125 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
+		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
+		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			is_streaming_ended = true;
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   originname,
+											   sizeof(originname),
+											   true);
+
+			/* Drop replication origin */
+			replorigin_drop_by_name(originname, true, false);
+		}
+
+		/*
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid))));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
 		StartTransactionCommand();
 
+		/*
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
+		 */
 		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
 										   originname,
-										   sizeof(originname));
+										   sizeof(originname),
+										   true);
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
-		 */
-		replorigin_drop_by_name(originname, true, false);
+		foreach(lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+			/*
+			 * Pick the table for the next run if there is not another worker
+			 * already picked that table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->is_first_run = false;
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->move_to_next_rel = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* Cleanup before next run or ending the worker. */
+		if (!MyLogicalRepWorker->move_to_next_rel)
+		{
+			/*
+			 * It is important to give an error if we are unable to drop the
+			 * slot, otherwise, it won't be dropped till the corresponding
+			 * subscription is dropped. So passing missing_ok = false.
+			 */
+			if (MyLogicalRepWorker->created_slot)
+			{
+				/* End streaming if it's not already ended. */
+				if (!is_streaming_ended)
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
 
-		finish_sync_worker();
+			/* Drop replication origin before exiting. */
+			replorigin_drop_by_name(originname, true, false);
+
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -485,6 +576,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -505,18 +597,27 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -605,11 +706,21 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
-												 rstate->relid);
+												 rstate->relid,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -802,6 +913,7 @@ fetch_remote_table_info(char *nspname, char *relname,
 		TupleTableSlot *tslot;
 		Oid			attrsRow[] = {INT2VECTOROID};
 		StringInfoData pub_names;
+
 		initStringInfo(&pub_names);
 		foreach(lc, MySubscription->publications)
 		{
@@ -1174,8 +1286,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1186,11 +1298,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%ld_" UINT64_FORMAT, suboid,
+			 slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1213,6 +1325,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1241,7 +1354,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1261,11 +1374,25 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc0(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
-									   MyLogicalRepWorker->relid,
 									   originname,
-									   sizeof(originname));
+									   sizeof(originname),
+									   true);
+
+	CommitTransactionCommand();
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
@@ -1280,10 +1407,48 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication drop might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for relation %u",
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1304,6 +1469,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		goto copy_table_done;
 	}
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1311,10 +1477,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1353,6 +1521,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1368,55 +1537,100 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
 	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
+	 *
 	 * Prevent cancel/die interrupts while creating slot here because it is
 	 * possible that before the server finishes this command, a concurrent
 	 * drop subscription happens which would complete without removing this
 	 * slot leading to a dangling slot on the server.
+	 *
 	 */
-	HOLD_INTERRUPTS();
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-	RESUME_INTERRUPTS();
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		HOLD_INTERRUPTS();
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		RESUME_INTERRUPTS();
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
 
-		replorigin_session_setup(originid);
-		replorigin_session_origin = originid;
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin wit the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		HOLD_INTERRUPTS();
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		RESUME_INTERRUPTS();
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot. This is WAL
+	 * logged for the purpose of recovery. Locks are to prevent the
+	 * replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1439,10 +1653,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index f9efe6c4c6..08349e8b5f 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -316,6 +316,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 							 bool first_segment);
 static void stream_write_change(char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -368,18 +369,26 @@ static inline void reset_apply_error_context_info(void);
  * Form the origin name for the subscription.
  *
  * This is a common function for tablesync and other workers. Tablesync workers
- * must pass a valid relid. Other callers must pass relid = InvalidOid.
+ * must pass is_tablesync true so that origin name includes slot id.
  *
  * Return the name in the supplied buffer.
  */
 void
-ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
-								   char *originname, Size szoriginname)
+ReplicationOriginNameForLogicalRep(Oid suboid, char *originname,
+								   Size szoriginname, bool is_tablesync)
 {
-	if (OidIsValid(relid))
+	if (is_tablesync)
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, MyLogicalRepWorker->relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%ld", suboid, MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -2834,6 +2843,23 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can move to next table while running
+				 * process_syncing_tables_for_sync. Before we were able to
+				 * reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of moving to next table. Now that tablesync workers
+				 * can be reused, we need to take care of memory contexts here
+				 * before moving to sync a table.
+				 */
+				if (MyLogicalRepWorker->move_to_next_rel)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -2853,6 +2879,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->move_to_next_rel)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -2954,8 +2984,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	/* Pop the error context stack */
 	error_context_stack = errcallback.previous;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's moving to next relation, this is a sync worker. Sync workers
+	 * end the streaming during process_syncing_tables_for_sync. Calling
+	 * endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->move_to_next_rel)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -3497,6 +3535,35 @@ stream_write_change(char action, StringInfo s)
 	BufFileWrite(stream_fd, &s->data[s->cursor], len);
 }
 
+/*
+ * stream_build_options_replication
+ * 		Build logical replication streaming options.
+ *
+ * This function sets streaming options including replication slot name
+ * and origin start position. Workers need these options for logical replication.
+ */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.streaming = MySubscription->stream;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -3571,6 +3638,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -3608,6 +3678,140 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->move_to_next_rel = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   originname,
+									   originname_size,
+									   true);
+	CommitTransactionCommand();
+
+	apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+															   originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard against
+	 * DDL bugs or manual catalog changes.  (libpqwalreceiver will crash if
+	 * slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, originname,
+									   originname_size, false);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+				 errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it does
+	 * some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+															   originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+}
+
 /* Logical Replication Apply worker entry point */
 void
 ApplyWorkerMain(Datum main_arg)
@@ -3618,7 +3822,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -3709,143 +3912,48 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		/*
-		 * Allocate the origin name in long-lived context for error context
-		 * message.
-		 */
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
-																   originname);
-	}
-	else
-	{
-		/* This is main apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		/*
-		 * Allocate the origin name in long-lived context for error context
-		 * message.
-		 */
-		apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
-																   originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-	options.proto.logical.streaming = MySubscription->stream;
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
-
-	if (!am_tablesync_worker())
+	/*
+	 * The loop where worker does its job. It loops until the worker is not
+	 * reused.
+	 */
+	while (MyLogicalRepWorker->is_first_run ||
+		   MyLogicalRepWorker->move_to_next_rel)
 	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		if (am_tablesync_worker())
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * This is a tablesync worker. Start syncing tables before
+			 * starting the apply loop.
+			 */
+			run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 		else
 		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* This is main apply worker */
+			run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+		/* Run the main loop. */
+		start_apply(origin_startpos);
 
+		if (MyLogicalRepWorker->move_to_next_rel)
+		{
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
+			CommitTransactionCommand();
+		}
+	}
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 7b98714f30..0b13dd34cb 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -102,6 +102,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -135,11 +138,14 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 8e88de7b2b..9cafb521f9 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	text		srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	text		srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,17 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 65f2c74239..aca6b590f9 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -219,8 +219,9 @@ extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 2b7114ff6d..e7f74e0f9f 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -30,6 +30,26 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if worker is running for the first time or in reuse
+	 */
+	bool		is_first_run;
+
+	/*
+	 * Indicates if the sync worker created a replication slot or it reuses an
+	 * existing one created by another worker.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -51,6 +71,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Used to indicate whether sync worker will be reused for another
+	 * relation
+	 */
+	bool		move_to_next_rel;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction, the
@@ -85,15 +111,15 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 												bool only_running);
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
-									 Oid userid, Oid relid);
+									 Oid userid, Oid relid, int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
 extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker);
 
 extern int	logicalrep_sync_worker_count(Oid subid);
 
-extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
-											   char *originname, Size szoriginname);
+extern void ReplicationOriginNameForLogicalRep(Oid suboid, char *originname,
+											   Size szoriginname, bool is_tablesync);
 extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos);
 
 extern bool AllTablesyncsReady(void);
diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out
index a57fd142a9..3d34a21421 100644
--- a/src/test/regress/expected/misc_sanity.out
+++ b/src/test/regress/expected/misc_sanity.out
@@ -47,20 +47,22 @@ WHERE c.oid < 16384 AND
       relkind = 'r' AND
       attstorage != 'p'
 ORDER BY 1, 2;
-         relname         |    attname    |   atttypid   
--------------------------+---------------+--------------
- pg_attribute            | attacl        | aclitem[]
- pg_attribute            | attfdwoptions | text[]
- pg_attribute            | attmissingval | anyarray
- pg_attribute            | attoptions    | text[]
- pg_class                | relacl        | aclitem[]
- pg_class                | reloptions    | text[]
- pg_class                | relpartbound  | pg_node_tree
- pg_index                | indexprs      | pg_node_tree
- pg_index                | indpred       | pg_node_tree
- pg_largeobject          | data          | bytea
- pg_largeobject_metadata | lomacl        | aclitem[]
-(11 rows)
+         relname         |     attname     |   atttypid   
+-------------------------+-----------------+--------------
+ pg_attribute            | attacl          | aclitem[]
+ pg_attribute            | attfdwoptions   | text[]
+ pg_attribute            | attmissingval   | anyarray
+ pg_attribute            | attoptions      | text[]
+ pg_class                | relacl          | aclitem[]
+ pg_class                | reloptions      | text[]
+ pg_class                | relpartbound    | pg_node_tree
+ pg_index                | indexprs        | pg_node_tree
+ pg_index                | indpred         | pg_node_tree
+ pg_largeobject          | data            | bytea
+ pg_largeobject_metadata | lomacl          | aclitem[]
+ pg_subscription_rel     | srreloriginname | text
+ pg_subscription_rel     | srrelslotname   | text
+(13 rows)
 
 -- system catalogs without primary keys
 --
-- 
2.25.1

From 9ffa22049582ac0464306d301b57aba8610376e7 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 1/2] Add replication protocol cmd to create a snapshot

Introduced REPLICATION_SLOT_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

REPLICATION_SLOT_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

REPLICATION_SLOT_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy a relation and then apply changes by consuming from replication slot.
---
 doc/src/sgml/protocol.sgml                    | 32 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       | 69 ++++++++++++-
 src/backend/replication/logical/logical.c     | 39 +++++++-
 .../replication/logical/logicalfuncs.c        |  1 +
 src/backend/replication/repl_gram.y           | 18 +++-
 src/backend/replication/repl_scanner.l        |  2 +
 src/backend/replication/slotfuncs.c           |  1 +
 src/backend/replication/walsender.c           | 97 ++++++++++++++++++-
 src/include/nodes/replnodes.h                 | 11 +++
 src/include/replication/logical.h             |  1 +
 src/include/replication/walreceiver.h         | 13 +++
 src/tools/pgindent/typedefs.list              |  1 +
 12 files changed, 281 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 03312e07e2..f4f29079ac 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2613,6 +2613,38 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>REPLICATION_SLOT_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>REPLICATION_SLOT_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the currenct transaction. This command is currently only supported
+       for logical replication.
+       slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index 7f697b0f29..e367c5f757 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -80,6 +80,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -96,7 +98,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -968,6 +971,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "REPLICATION_SLOT_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 625a7f4273..16c0a27ba5 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -461,6 +461,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, create a snapshot able to read all tables,
+ * 		otherwise do not create any snapshot.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -479,6 +483,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -487,6 +492,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -533,8 +539,39 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 5c23178570..e4b80f2764 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 7ba33d6672..f70b2e087a 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_REPLICATION_SLOT_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show replication_slot_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| replication_slot_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * REPLICATION_SLOT_SNAPSHOT %s options
+ */
+replication_slot_snapshot:
+			K_REPLICATION_SLOT_SNAPSHOT var_name plugin_options
+				{
+					ReplicationSlotSnapshotCmd *n = makeNode(ReplicationSlotSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_REPLICATION_SLOT_SNAPSHOT	{ $$ = "replication_slot_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index 72ef12225e..e33030da61 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+REPLICATION_SLOT_SNAPSHOT	{ return K_REPLICATION_SLOT_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_REPLICATION_SLOT_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 79d7f1372d..13d099fe06 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -478,6 +478,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c11bb3716f..f9a22c8b16 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void ReplicationSlotSnapshot(ReplicationSlotSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1280,7 +1281,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1332,6 +1333,91 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+ReplicationSlotSnapshot(ReplicationSlotSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the replication slot and don't yet accept feedback messages or send
+	 * keepalives. As we possibly need to wait for further WAL the walsender
+	 * would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consisten WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1860,6 +1946,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_ReplicationSlotSnapshotCmd:
+			{
+				cmdtag = "REPLICATION_SLOT_SNAPSHOT";
+				set_ps_display(cmdtag);
+				ReplicationSlotSnapshot((ReplicationSlotSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 8ae9c517ff..94aa7d2253 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		REPLICATION_SLOT_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct ReplicationSlotSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} ReplicationSlotSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 4ad019e25a..3875b32f9d 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 9339f29303..274a162b48 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -384,6 +384,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -401,6 +411,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -435,6 +446,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 60c71d05fe..6b90a4a693 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2315,6 +2315,7 @@ ReplicationSlotCtlData
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+ReplicationSlotSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
-- 
2.25.1

From ce74b9a18a2a425ed39533f435fefaa2431285c2 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 2/2] Reuse Logical Replication Background worker

This commit allows tablesync workers to move to another table that needs synchronization,
when they're done with the current table in tablesync phase of Logical Replication.

Before this commit, tablesync workers were capable of syncing only one
relation. A new worker, replication slot and origin were needed for each
relation in the subscription.

Now, tablesync workers are not only limited with one relation and can move to another relation and reuse existing
replication slots and origins

This reduces the overhead of launching/killing a new background worker for each relation.
By reusing tablesync workers, replication slots and origins created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot/origin improves tablesync speed significantly especially for empty or small tables.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                  |  29 ++
 src/backend/catalog/pg_subscription.c       | 284 ++++++++++++-
 src/backend/commands/subscriptioncmds.c     | 238 +++++++----
 src/backend/replication/logical/launcher.c  |   9 +-
 src/backend/replication/logical/tablesync.c | 434 +++++++++++++++-----
 src/backend/replication/logical/worker.c    | 370 +++++++++++------
 src/include/catalog/pg_subscription.h       |   6 +
 src/include/catalog/pg_subscription_rel.h   |  15 +-
 src/include/replication/slot.h              |   3 +-
 src/include/replication/worker_internal.h   |  32 +-
 src/test/regress/expected/misc_sanity.out   |  30 +-
 11 files changed, 1095 insertions(+), 355 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 9316b811ac..7d55db6354 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7997,6 +7997,17 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+       The last used ID for tablesync workers. This ID is used to
+       create replication slots. The last used ID needs to be stored
+       to make logical replication can safely proceed after any interruption.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8081,6 +8092,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>text</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>text</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a506fc3ec8..1f637df242 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -114,6 +114,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +213,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +280,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,6 +309,14 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -275,6 +329,58 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +411,48 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +464,134 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Get origin name of subscription table.
+ *
+ * Returns null if the subscription table does not have a origin.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = TextDatumGetCString(d);
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * Returns null if the subscription table does not have a replication slot.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = TextDatumGetCString(d);
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index d673557ea4..ccdd844201 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -648,6 +648,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(1);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -657,7 +658,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 
 	recordDependencyOnOwner(SubscriptionRelationId, subid, owner);
 
-	ReplicationOriginNameForLogicalRep(subid, InvalidOid, originname, sizeof(originname));
+	ReplicationOriginNameForLogicalRep(subid, originname, sizeof(originname), false);
 	replorigin_create(originname);
 
 	/*
@@ -708,7 +709,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -798,6 +799,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -875,7 +878,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -899,6 +902,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -925,7 +929,29 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker if exists store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -945,8 +971,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 					 * origin and by this time the origin might be already
 					 * removed. For these reasons, passing missing_ok = true.
 					 */
-					ReplicationOriginNameForLogicalRep(sub->oid, relid, originname,
-													   sizeof(originname));
+					ReplicationOriginNameForLogicalRep(sub->oid, originname,
+													   sizeof(originname), true);
 					replorigin_drop_by_name(originname, true, false);
 				}
 
@@ -959,31 +985,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1319,8 +1338,8 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt,
 					char		originname[NAMEDATALEN];
 					XLogRecPtr	remote_lsn;
 
-					ReplicationOriginNameForLogicalRep(subid, InvalidOid,
-													   originname, sizeof(originname));
+					ReplicationOriginNameForLogicalRep(subid, originname,
+													   sizeof(originname), false);
 					originid = replorigin_by_name(originname, false);
 					remote_lsn = replorigin_get_progress(originid, false);
 
@@ -1380,6 +1399,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1451,6 +1471,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1500,37 +1528,29 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%ld", subid, i);
+		/* missin_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1538,7 +1558,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	RemoveSubscriptionRel(subid, InvalidOid);
 
 	/* Remove the origin tracking if exists. */
-	ReplicationOriginNameForLogicalRep(subid, InvalidOid, originname, sizeof(originname));
+	ReplicationOriginNameForLogicalRep(subid, originname, sizeof(originname), false);
 	replorigin_drop_by_name(originname, true, false);
 
 	/*
@@ -1582,39 +1602,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1637,6 +1635,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("not tuple returned."));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+	\
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -1989,6 +2052,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -1999,18 +2063,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 73594c698a..5b5dead996 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -264,7 +264,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 void
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid)
+						 Oid relid, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -371,7 +371,11 @@ retry:
 	/* Prepare the worker slot. */
 	worker->launch_time = now;
 	worker->in_use = true;
+	worker->is_first_run = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -379,6 +383,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->move_to_next_rel = false;
 	worker->stream_fileset = NULL;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
@@ -869,7 +874,7 @@ ApplyLauncherMain(Datum main_arg)
 					wait_time = wal_retrieve_retry_interval;
 
 					logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
-											 sub->owner, InvalidOid);
+											 sub->owner, InvalidOid, 0);
 				}
 			}
 
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 94e813ac53..a6364d9433 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -126,12 +126,8 @@ static bool FetchTableStates(bool *started_tx);
 
 static StringInfo copybuf = NULL;
 
-/*
- * Exit routine for synchronization worker.
- */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +139,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the main apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -284,6 +290,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -292,6 +302,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		TimeLineID	tli;
 		char		syncslotname[NAMEDATALEN] = {0};
 		char		originname[NAMEDATALEN] = {0};
+		bool		is_streaming_ended = false;
 
 		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
 		MyLogicalRepWorker->relstate_lsn = current_lsn;
@@ -308,40 +319,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication trackin origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -350,34 +350,125 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
+		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
+		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			is_streaming_ended = true;
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   originname,
+											   sizeof(originname),
+											   true);
+
+			/* Drop replication origin */
+			replorigin_drop_by_name(originname, true, false);
+		}
+
+		/*
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid))));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
 		StartTransactionCommand();
 
+		/*
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
+		 */
 		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
 										   originname,
-										   sizeof(originname));
+										   sizeof(originname),
+										   true);
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
-		 */
-		replorigin_drop_by_name(originname, true, false);
+		foreach(lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+			/*
+			 * Pick the table for the next run if there is not another worker
+			 * already picked that table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->is_first_run = false;
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->move_to_next_rel = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* Cleanup before next run or ending the worker. */
+		if (!MyLogicalRepWorker->move_to_next_rel)
+		{
+			/*
+			 * It is important to give an error if we are unable to drop the
+			 * slot, otherwise, it won't be dropped till the corresponding
+			 * subscription is dropped. So passing missing_ok = false.
+			 */
+			if (MyLogicalRepWorker->created_slot)
+			{
+				/* End streaming if it's not already ended. */
+				if (!is_streaming_ended)
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
 
-		finish_sync_worker();
+			/* Drop replication origin before exiting. */
+			replorigin_drop_by_name(originname, true, false);
+
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -485,6 +576,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -505,18 +597,27 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -605,11 +706,21 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
-												 rstate->relid);
+												 rstate->relid,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -802,6 +913,7 @@ fetch_remote_table_info(char *nspname, char *relname,
 		TupleTableSlot *tslot;
 		Oid			attrsRow[] = {INT2VECTOROID};
 		StringInfoData pub_names;
+
 		initStringInfo(&pub_names);
 		foreach(lc, MySubscription->publications)
 		{
@@ -1174,8 +1286,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1186,11 +1298,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%ld_" UINT64_FORMAT, suboid,
+			 slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1213,6 +1325,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1241,7 +1354,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1261,11 +1374,25 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc0(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
-									   MyLogicalRepWorker->relid,
 									   originname,
-									   sizeof(originname));
+									   sizeof(originname),
+									   true);
+
+	CommitTransactionCommand();
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
@@ -1280,10 +1407,48 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication drop might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for relation %u",
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1304,6 +1469,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		goto copy_table_done;
 	}
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1311,10 +1477,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1353,6 +1521,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1368,55 +1537,100 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
 	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
+	 *
 	 * Prevent cancel/die interrupts while creating slot here because it is
 	 * possible that before the server finishes this command, a concurrent
 	 * drop subscription happens which would complete without removing this
 	 * slot leading to a dangling slot on the server.
+	 *
 	 */
-	HOLD_INTERRUPTS();
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-	RESUME_INTERRUPTS();
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		HOLD_INTERRUPTS();
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		RESUME_INTERRUPTS();
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
 
-		replorigin_session_setup(originid);
-		replorigin_session_origin = originid;
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin wit the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		HOLD_INTERRUPTS();
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		RESUME_INTERRUPTS();
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot. This is WAL
+	 * logged for the purpose of recovery. Locks are to prevent the
+	 * replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1439,10 +1653,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 96772e4d73..0b325b8fa0 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -317,6 +317,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 							 bool first_segment);
 static void stream_write_change(char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -369,18 +370,26 @@ static inline void reset_apply_error_context_info(void);
  * Form the origin name for the subscription.
  *
  * This is a common function for tablesync and other workers. Tablesync workers
- * must pass a valid relid. Other callers must pass relid = InvalidOid.
+ * must pass is_tablesync true so that origin name includes slot id.
  *
  * Return the name in the supplied buffer.
  */
 void
-ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
-								   char *originname, Size szoriginname)
+ReplicationOriginNameForLogicalRep(Oid suboid, char *originname,
+								   Size szoriginname, bool is_tablesync)
 {
-	if (OidIsValid(relid))
+	if (is_tablesync)
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, MyLogicalRepWorker->relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%ld", suboid, MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -2839,6 +2848,23 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can move to next table while running
+				 * process_syncing_tables_for_sync. Before we were able to
+				 * reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of moving to next table. Now that tablesync workers
+				 * can be reused, we need to take care of memory contexts here
+				 * before moving to sync a table.
+				 */
+				if (MyLogicalRepWorker->move_to_next_rel)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -2858,6 +2884,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->move_to_next_rel)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -2959,8 +2989,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	/* Pop the error context stack */
 	error_context_stack = errcallback.previous;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's moving to next relation, this is a sync worker. Sync workers
+	 * end the streaming during process_syncing_tables_for_sync. Calling
+	 * endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->move_to_next_rel)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -3502,6 +3540,35 @@ stream_write_change(char action, StringInfo s)
 	BufFileWrite(stream_fd, &s->data[s->cursor], len);
 }
 
+/*
+ * stream_build_options_replication
+ * 		Build logical replication streaming options.
+ *
+ * This function sets streaming options including replication slot name
+ * and origin start position. Workers need these options for logical replication.
+ */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.streaming = MySubscription->stream;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -3576,6 +3643,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -3613,6 +3683,140 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->move_to_next_rel = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   originname,
+									   originname_size,
+									   true);
+	CommitTransactionCommand();
+
+	apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+															   originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard against
+	 * DDL bugs or manual catalog changes.  (libpqwalreceiver will crash if
+	 * slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, originname,
+									   originname_size, false);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+				 errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it does
+	 * some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+															   originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+}
+
 /* Logical Replication Apply worker entry point */
 void
 ApplyWorkerMain(Datum main_arg)
@@ -3623,7 +3827,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -3714,143 +3917,48 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		/*
-		 * Allocate the origin name in long-lived context for error context
-		 * message.
-		 */
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
-																   originname);
-	}
-	else
-	{
-		/* This is main apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		/*
-		 * Allocate the origin name in long-lived context for error context
-		 * message.
-		 */
-		apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
-																   originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-	options.proto.logical.streaming = MySubscription->stream;
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
-
-	if (!am_tablesync_worker())
+	/*
+	 * The loop where worker does its job. It loops until the worker is not
+	 * reused.
+	 */
+	while (MyLogicalRepWorker->is_first_run ||
+		   MyLogicalRepWorker->move_to_next_rel)
 	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		if (am_tablesync_worker())
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * This is a tablesync worker. Start syncing tables before
+			 * starting the apply loop.
+			 */
+			run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 		else
 		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* This is main apply worker */
+			run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+		/* Run the main loop. */
+		start_apply(origin_startpos);
 
+		if (MyLogicalRepWorker->move_to_next_rel)
+		{
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
+			CommitTransactionCommand();
+		}
+	}
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 7b98714f30..0b13dd34cb 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -102,6 +102,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -135,11 +138,14 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 8e88de7b2b..9cafb521f9 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	text		srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	text		srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,17 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 65f2c74239..aca6b590f9 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -219,8 +219,9 @@ extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 2b7114ff6d..e7f74e0f9f 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -30,6 +30,26 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if worker is running for the first time or in reuse
+	 */
+	bool		is_first_run;
+
+	/*
+	 * Indicates if the sync worker created a replication slot or it reuses an
+	 * existing one created by another worker.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -51,6 +71,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Used to indicate whether sync worker will be reused for another
+	 * relation
+	 */
+	bool		move_to_next_rel;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction, the
@@ -85,15 +111,15 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 												bool only_running);
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
-									 Oid userid, Oid relid);
+									 Oid userid, Oid relid, int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
 extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker);
 
 extern int	logicalrep_sync_worker_count(Oid subid);
 
-extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
-											   char *originname, Size szoriginname);
+extern void ReplicationOriginNameForLogicalRep(Oid suboid, char *originname,
+											   Size szoriginname, bool is_tablesync);
 extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos);
 
 extern bool AllTablesyncsReady(void);
diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out
index a57fd142a9..3d34a21421 100644
--- a/src/test/regress/expected/misc_sanity.out
+++ b/src/test/regress/expected/misc_sanity.out
@@ -47,20 +47,22 @@ WHERE c.oid < 16384 AND
       relkind = 'r' AND
       attstorage != 'p'
 ORDER BY 1, 2;
-         relname         |    attname    |   atttypid   
--------------------------+---------------+--------------
- pg_attribute            | attacl        | aclitem[]
- pg_attribute            | attfdwoptions | text[]
- pg_attribute            | attmissingval | anyarray
- pg_attribute            | attoptions    | text[]
- pg_class                | relacl        | aclitem[]
- pg_class                | reloptions    | text[]
- pg_class                | relpartbound  | pg_node_tree
- pg_index                | indexprs      | pg_node_tree
- pg_index                | indpred       | pg_node_tree
- pg_largeobject          | data          | bytea
- pg_largeobject_metadata | lomacl        | aclitem[]
-(11 rows)
+         relname         |     attname     |   atttypid   
+-------------------------+-----------------+--------------
+ pg_attribute            | attacl          | aclitem[]
+ pg_attribute            | attfdwoptions   | text[]
+ pg_attribute            | attmissingval   | anyarray
+ pg_attribute            | attoptions      | text[]
+ pg_class                | relacl          | aclitem[]
+ pg_class                | reloptions      | text[]
+ pg_class                | relpartbound    | pg_node_tree
+ pg_index                | indexprs        | pg_node_tree
+ pg_index                | indpred         | pg_node_tree
+ pg_largeobject          | data            | bytea
+ pg_largeobject_metadata | lomacl          | aclitem[]
+ pg_subscription_rel     | srreloriginname | text
+ pg_subscription_rel     | srrelslotname   | text
+(13 rows)
 
 -- system catalogs without primary keys
 --
-- 
2.25.1

From 9ffa22049582ac0464306d301b57aba8610376e7 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 1/2] Add replication protocol cmd to create a snapshot

Introduced REPLICATION_SLOT_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

REPLICATION_SLOT_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

REPLICATION_SLOT_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy a relation and then apply changes by consuming from replication slot.
---
 doc/src/sgml/protocol.sgml                    | 32 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       | 69 ++++++++++++-
 src/backend/replication/logical/logical.c     | 39 +++++++-
 .../replication/logical/logicalfuncs.c        |  1 +
 src/backend/replication/repl_gram.y           | 18 +++-
 src/backend/replication/repl_scanner.l        |  2 +
 src/backend/replication/slotfuncs.c           |  1 +
 src/backend/replication/walsender.c           | 97 ++++++++++++++++++-
 src/include/nodes/replnodes.h                 | 11 +++
 src/include/replication/logical.h             |  1 +
 src/include/replication/walreceiver.h         | 13 +++
 src/tools/pgindent/typedefs.list              |  1 +
 12 files changed, 281 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 03312e07e2..f4f29079ac 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2613,6 +2613,38 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>REPLICATION_SLOT_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>REPLICATION_SLOT_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the currenct transaction. This command is currently only supported
+       for logical replication.
+       slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index 7f697b0f29..e367c5f757 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -80,6 +80,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -96,7 +98,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -968,6 +971,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "REPLICATION_SLOT_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 625a7f4273..16c0a27ba5 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -461,6 +461,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, create a snapshot able to read all tables,
+ * 		otherwise do not create any snapshot.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -479,6 +483,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -487,6 +492,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -533,8 +539,39 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 5c23178570..e4b80f2764 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 7ba33d6672..f70b2e087a 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_REPLICATION_SLOT_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show replication_slot_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| replication_slot_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * REPLICATION_SLOT_SNAPSHOT %s options
+ */
+replication_slot_snapshot:
+			K_REPLICATION_SLOT_SNAPSHOT var_name plugin_options
+				{
+					ReplicationSlotSnapshotCmd *n = makeNode(ReplicationSlotSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_REPLICATION_SLOT_SNAPSHOT	{ $$ = "replication_slot_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index 72ef12225e..e33030da61 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+REPLICATION_SLOT_SNAPSHOT	{ return K_REPLICATION_SLOT_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_REPLICATION_SLOT_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 79d7f1372d..13d099fe06 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -478,6 +478,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index c11bb3716f..f9a22c8b16 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void ReplicationSlotSnapshot(ReplicationSlotSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1280,7 +1281,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1332,6 +1333,91 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+ReplicationSlotSnapshot(ReplicationSlotSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the replication slot and don't yet accept feedback messages or send
+	 * keepalives. As we possibly need to wait for further WAL the walsender
+	 * would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consisten WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1860,6 +1946,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_ReplicationSlotSnapshotCmd:
+			{
+				cmdtag = "REPLICATION_SLOT_SNAPSHOT";
+				set_ps_display(cmdtag);
+				ReplicationSlotSnapshot((ReplicationSlotSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 8ae9c517ff..94aa7d2253 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		REPLICATION_SLOT_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct ReplicationSlotSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} ReplicationSlotSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 4ad019e25a..3875b32f9d 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 9339f29303..274a162b48 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -384,6 +384,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -401,6 +411,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -435,6 +446,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 60c71d05fe..6b90a4a693 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2315,6 +2315,7 @@ ReplicationSlotCtlData
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+ReplicationSlotSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
-- 
2.25.1

From 901624f1d48af5a2766af9ba85fd28ccffd0daef Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 2/2] Reuse Logical Replication Background worker

This commit allows tablesync workers to move to another table that needs synchronization,
when they're done with the current table in tablesync phase of Logical Replication.

Before this commit, tablesync workers were capable of syncing only one
relation. A new worker, replication slot and origin were needed for each
relation in the subscription.

Now, tablesync workers are not only limited with one relation and can move to another relation and reuse existing
replication slots and origins

This reduces the overhead of launching/killing a new background worker for each relation.
By reusing tablesync workers, replication slots and origins created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot/origin improves tablesync speed significantly especially for empty or small tables.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                  |  29 ++
 src/backend/catalog/pg_subscription.c       | 284 ++++++++++++-
 src/backend/commands/subscriptioncmds.c     | 238 +++++++----
 src/backend/replication/logical/launcher.c  |   9 +-
 src/backend/replication/logical/tablesync.c | 434 +++++++++++++++-----
 src/backend/replication/logical/worker.c    | 370 +++++++++++------
 src/include/catalog/pg_subscription.h       |   6 +
 src/include/catalog/pg_subscription_rel.h   |  15 +-
 src/include/replication/slot.h              |   3 +-
 src/include/replication/worker_internal.h   |  32 +-
 src/test/regress/expected/misc_sanity.out   |  30 +-
 11 files changed, 1095 insertions(+), 355 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 9316b811ac..7d55db6354 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -7997,6 +7997,17 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+       The last used ID for tablesync workers. This ID is used to
+       create replication slots. The last used ID needs to be stored
+       to make logical replication can safely proceed after any interruption.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8081,6 +8092,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>text</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>text</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a56ae311c3..98d627ff49 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -114,6 +114,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +213,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +280,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,6 +309,14 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -275,6 +329,58 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +411,48 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +464,134 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Get origin name of subscription table.
+ *
+ * Returns null if the subscription table does not have a origin.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = TextDatumGetCString(d);
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * Returns null if the subscription table does not have a replication slot.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = TextDatumGetCString(d);
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index b9c5df796f..964b698e61 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -648,6 +648,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(1);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -657,7 +658,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 
 	recordDependencyOnOwner(SubscriptionRelationId, subid, owner);
 
-	ReplicationOriginNameForLogicalRep(subid, InvalidOid, originname, sizeof(originname));
+	ReplicationOriginNameForLogicalRep(subid, originname, sizeof(originname), false);
 	replorigin_create(originname);
 
 	/*
@@ -708,7 +709,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -798,6 +799,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -875,7 +878,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -899,6 +902,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -925,7 +929,29 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker if exists store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -945,8 +971,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 					 * origin and by this time the origin might be already
 					 * removed. For these reasons, passing missing_ok = true.
 					 */
-					ReplicationOriginNameForLogicalRep(sub->oid, relid, originname,
-													   sizeof(originname));
+					ReplicationOriginNameForLogicalRep(sub->oid, originname,
+													   sizeof(originname), true);
 					replorigin_drop_by_name(originname, true, false);
 				}
 
@@ -959,31 +985,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1319,8 +1338,8 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt,
 					char		originname[NAMEDATALEN];
 					XLogRecPtr	remote_lsn;
 
-					ReplicationOriginNameForLogicalRep(subid, InvalidOid,
-													   originname, sizeof(originname));
+					ReplicationOriginNameForLogicalRep(subid, originname,
+													   sizeof(originname), false);
 					originid = replorigin_by_name(originname, false);
 					remote_lsn = replorigin_get_progress(originid, false);
 
@@ -1380,6 +1399,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1451,6 +1471,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1500,37 +1528,29 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missin_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1538,7 +1558,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	RemoveSubscriptionRel(subid, InvalidOid);
 
 	/* Remove the origin tracking if exists. */
-	ReplicationOriginNameForLogicalRep(subid, InvalidOid, originname, sizeof(originname));
+	ReplicationOriginNameForLogicalRep(subid, originname, sizeof(originname), false);
 	replorigin_drop_by_name(originname, true, false);
 
 	/*
@@ -1582,39 +1602,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1637,6 +1635,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("not tuple returned."));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+	\
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -1989,6 +2052,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -1999,18 +2063,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index a69e371c05..0d33d3592b 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -264,7 +264,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 void
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid)
+						 Oid relid, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -371,7 +371,11 @@ retry:
 	/* Prepare the worker slot. */
 	worker->launch_time = now;
 	worker->in_use = true;
+	worker->is_first_run = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -379,6 +383,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->move_to_next_rel = false;
 	worker->stream_fileset = NULL;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
@@ -869,7 +874,7 @@ ApplyLauncherMain(Datum main_arg)
 					wait_time = wal_retrieve_retry_interval;
 
 					logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
-											 sub->owner, InvalidOid);
+											 sub->owner, InvalidOid, 0);
 				}
 			}
 
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 2fdfeb5b4c..bb242e0c95 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -126,12 +126,8 @@ static bool FetchTableStates(bool *started_tx);
 
 static StringInfo copybuf = NULL;
 
-/*
- * Exit routine for synchronization worker.
- */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +139,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the main apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -284,6 +290,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -292,6 +302,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		TimeLineID	tli;
 		char		syncslotname[NAMEDATALEN] = {0};
 		char		originname[NAMEDATALEN] = {0};
+		bool		is_streaming_ended = false;
 
 		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
 		MyLogicalRepWorker->relstate_lsn = current_lsn;
@@ -308,40 +319,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication trackin origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -350,34 +350,125 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
+		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
+		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			is_streaming_ended = true;
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   originname,
+											   sizeof(originname),
+											   true);
+
+			/* Drop replication origin */
+			replorigin_drop_by_name(originname, true, false);
+		}
+
+		/*
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid))));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
 		StartTransactionCommand();
 
+		/*
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
+		 */
 		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
 										   originname,
-										   sizeof(originname));
+										   sizeof(originname),
+										   true);
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
-		 */
-		replorigin_drop_by_name(originname, true, false);
+		foreach(lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+			/*
+			 * Pick the table for the next run if there is not another worker
+			 * already picked that table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->is_first_run = false;
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->move_to_next_rel = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* Cleanup before next run or ending the worker. */
+		if (!MyLogicalRepWorker->move_to_next_rel)
+		{
+			/*
+			 * It is important to give an error if we are unable to drop the
+			 * slot, otherwise, it won't be dropped till the corresponding
+			 * subscription is dropped. So passing missing_ok = false.
+			 */
+			if (MyLogicalRepWorker->created_slot)
+			{
+				/* End streaming if it's not already ended. */
+				if (!is_streaming_ended)
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
 
-		finish_sync_worker();
+			/* Drop replication origin before exiting. */
+			replorigin_drop_by_name(originname, true, false);
+
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -485,6 +576,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -505,18 +597,27 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -605,11 +706,21 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
-												 rstate->relid);
+												 rstate->relid,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -802,6 +913,7 @@ fetch_remote_table_info(char *nspname, char *relname,
 		TupleTableSlot *tslot;
 		Oid			attrsRow[] = {INT2VECTOROID};
 		StringInfoData pub_names;
+
 		initStringInfo(&pub_names);
 		foreach(lc, MySubscription->publications)
 		{
@@ -1174,8 +1286,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1186,11 +1298,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1213,6 +1325,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1241,7 +1354,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1261,11 +1374,25 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc0(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
-									   MyLogicalRepWorker->relid,
 									   originname,
-									   sizeof(originname));
+									   sizeof(originname),
+									   true);
+
+	CommitTransactionCommand();
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
@@ -1280,10 +1407,48 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication drop might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for relation %u",
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1304,6 +1469,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		goto copy_table_done;
 	}
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1311,10 +1477,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1353,6 +1521,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1368,55 +1537,100 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
 	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
+	 *
 	 * Prevent cancel/die interrupts while creating slot here because it is
 	 * possible that before the server finishes this command, a concurrent
 	 * drop subscription happens which would complete without removing this
 	 * slot leading to a dangling slot on the server.
+	 *
 	 */
-	HOLD_INTERRUPTS();
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-	RESUME_INTERRUPTS();
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		HOLD_INTERRUPTS();
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		RESUME_INTERRUPTS();
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
 
-		replorigin_session_setup(originid);
-		replorigin_session_origin = originid;
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin wit the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		HOLD_INTERRUPTS();
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		RESUME_INTERRUPTS();
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot. This is WAL
+	 * logged for the purpose of recovery. Locks are to prevent the
+	 * replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1439,10 +1653,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 446f84fa97..b10660384a 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -317,6 +317,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 							 bool first_segment);
 static void stream_write_change(char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -369,18 +370,26 @@ static inline void reset_apply_error_context_info(void);
  * Form the origin name for the subscription.
  *
  * This is a common function for tablesync and other workers. Tablesync workers
- * must pass a valid relid. Other callers must pass relid = InvalidOid.
+ * must pass is_tablesync true so that origin name includes slot id.
  *
  * Return the name in the supplied buffer.
  */
 void
-ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
-								   char *originname, Size szoriginname)
+ReplicationOriginNameForLogicalRep(Oid suboid, char *originname,
+								   Size szoriginname, bool is_tablesync)
 {
-	if (OidIsValid(relid))
+	if (is_tablesync)
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, MyLogicalRepWorker->relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -2839,6 +2848,23 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can move to next table while running
+				 * process_syncing_tables_for_sync. Before we were able to
+				 * reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of moving to next table. Now that tablesync workers
+				 * can be reused, we need to take care of memory contexts here
+				 * before moving to sync a table.
+				 */
+				if (MyLogicalRepWorker->move_to_next_rel)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -2858,6 +2884,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->move_to_next_rel)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -2959,8 +2989,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	/* Pop the error context stack */
 	error_context_stack = errcallback.previous;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's moving to next relation, this is a sync worker. Sync workers
+	 * end the streaming during process_syncing_tables_for_sync. Calling
+	 * endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->move_to_next_rel)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -3502,6 +3540,35 @@ stream_write_change(char action, StringInfo s)
 	BufFileWrite(stream_fd, &s->data[s->cursor], len);
 }
 
+/*
+ * stream_build_options_replication
+ * 		Build logical replication streaming options.
+ *
+ * This function sets streaming options including replication slot name
+ * and origin start position. Workers need these options for logical replication.
+ */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.streaming = MySubscription->stream;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -3576,6 +3643,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -3613,6 +3683,140 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->move_to_next_rel = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   originname,
+									   originname_size,
+									   true);
+	CommitTransactionCommand();
+
+	apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+															   originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard against
+	 * DDL bugs or manual catalog changes.  (libpqwalreceiver will crash if
+	 * slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+				 errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, originname,
+									   originname_size, false);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+				 errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it does
+	 * some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
+															   originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+}
+
 /* Logical Replication Apply worker entry point */
 void
 ApplyWorkerMain(Datum main_arg)
@@ -3623,7 +3827,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -3714,143 +3917,48 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		/*
-		 * Allocate the origin name in long-lived context for error context
-		 * message.
-		 */
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
-																   originname);
-	}
-	else
-	{
-		/* This is main apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		/*
-		 * Allocate the origin name in long-lived context for error context
-		 * message.
-		 */
-		apply_error_callback_arg.origin_name = MemoryContextStrdup(ApplyContext,
-																   originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-	options.proto.logical.streaming = MySubscription->stream;
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
-
-	if (!am_tablesync_worker())
+	/*
+	 * The loop where worker does its job. It loops until the worker is not
+	 * reused.
+	 */
+	while (MyLogicalRepWorker->is_first_run ||
+		   MyLogicalRepWorker->move_to_next_rel)
 	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		if (am_tablesync_worker())
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * This is a tablesync worker. Start syncing tables before
+			 * starting the apply loop.
+			 */
+			run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 		else
 		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* This is main apply worker */
+			run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+		/* Run the main loop. */
+		start_apply(origin_startpos);
 
+		if (MyLogicalRepWorker->move_to_next_rel)
+		{
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
+			CommitTransactionCommand();
+		}
+	}
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index eb4eeb448b..326be6cd5b 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -102,6 +102,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -135,11 +138,14 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..a35d04cccd 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	text		srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	text		srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,17 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8872c80cdf..3547daaaec 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -219,8 +219,9 @@ extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 2a3ec5c2d8..563cc9637a 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -30,6 +30,26 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if worker is running for the first time or in reuse
+	 */
+	bool		is_first_run;
+
+	/*
+	 * Indicates if the sync worker created a replication slot or it reuses an
+	 * existing one created by another worker.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -51,6 +71,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Used to indicate whether sync worker will be reused for another
+	 * relation
+	 */
+	bool		move_to_next_rel;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction, the
@@ -85,15 +111,15 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 												bool only_running);
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern void logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
-									 Oid userid, Oid relid);
+									 Oid userid, Oid relid, int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
 extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker);
 
 extern int	logicalrep_sync_worker_count(Oid subid);
 
-extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
-											   char *originname, Size szoriginname);
+extern void ReplicationOriginNameForLogicalRep(Oid suboid, char *originname,
+											   Size szoriginname, bool is_tablesync);
 extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos);
 
 extern bool AllTablesyncsReady(void);
diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out
index a57fd142a9..3d34a21421 100644
--- a/src/test/regress/expected/misc_sanity.out
+++ b/src/test/regress/expected/misc_sanity.out
@@ -47,20 +47,22 @@ WHERE c.oid < 16384 AND
       relkind = 'r' AND
       attstorage != 'p'
 ORDER BY 1, 2;
-         relname         |    attname    |   atttypid   
--------------------------+---------------+--------------
- pg_attribute            | attacl        | aclitem[]
- pg_attribute            | attfdwoptions | text[]
- pg_attribute            | attmissingval | anyarray
- pg_attribute            | attoptions    | text[]
- pg_class                | relacl        | aclitem[]
- pg_class                | reloptions    | text[]
- pg_class                | relpartbound  | pg_node_tree
- pg_index                | indexprs      | pg_node_tree
- pg_index                | indpred       | pg_node_tree
- pg_largeobject          | data          | bytea
- pg_largeobject_metadata | lomacl        | aclitem[]
-(11 rows)
+         relname         |     attname     |   atttypid   
+-------------------------+-----------------+--------------
+ pg_attribute            | attacl          | aclitem[]
+ pg_attribute            | attfdwoptions   | text[]
+ pg_attribute            | attmissingval   | anyarray
+ pg_attribute            | attoptions      | text[]
+ pg_class                | relacl          | aclitem[]
+ pg_class                | reloptions      | text[]
+ pg_class                | relpartbound    | pg_node_tree
+ pg_index                | indexprs        | pg_node_tree
+ pg_index                | indpred         | pg_node_tree
+ pg_largeobject          | data            | bytea
+ pg_largeobject_metadata | lomacl          | aclitem[]
+ pg_subscription_rel     | srreloriginname | text
+ pg_subscription_rel     | srrelslotname   | text
+(13 rows)
 
 -- system catalogs without primary keys
 --
-- 
2.25.1

From 0392f3e3f5c371e0b11c853a419f48b5e402660b Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 1/2] Add replication protocol cmd to create a snapshot

Introduced REPLICATION_SLOT_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

REPLICATION_SLOT_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

REPLICATION_SLOT_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy a relation and then apply changes by consuming from replication slot.
---
 doc/src/sgml/protocol.sgml                    | 32 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       | 69 ++++++++++++-
 src/backend/replication/logical/logical.c     | 39 +++++++-
 .../replication/logical/logicalfuncs.c        |  1 +
 src/backend/replication/repl_gram.y           | 18 +++-
 src/backend/replication/repl_scanner.l        |  2 +
 src/backend/replication/slotfuncs.c           |  1 +
 src/backend/replication/walsender.c           | 97 ++++++++++++++++++-
 src/include/nodes/replnodes.h                 | 11 +++
 src/include/replication/logical.h             |  1 +
 src/include/replication/walreceiver.h         | 13 +++
 src/tools/pgindent/typedefs.list              |  1 +
 12 files changed, 281 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 93fc7167d4..93a3867996 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2613,6 +2613,38 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>REPLICATION_SLOT_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>REPLICATION_SLOT_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the currenct transaction. This command is currently only supported
+       for logical replication.
+       slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index c40c6220db..9213fd2b1b 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -80,6 +80,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -96,7 +98,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -968,6 +971,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "REPLICATION_SLOT_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 52d1fe6269..b62babe359 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -461,6 +461,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, create a snapshot able to read all tables,
+ * 		otherwise do not create any snapshot.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -479,6 +483,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -487,6 +492,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -533,8 +539,39 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index fa1b641a2b..1191c70eb0 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..e5f0235d1e 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_REPLICATION_SLOT_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show replication_slot_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| replication_slot_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * REPLICATION_SLOT_SNAPSHOT %s options
+ */
+replication_slot_snapshot:
+			K_REPLICATION_SLOT_SNAPSHOT var_name plugin_options
+				{
+					ReplicationSlotSnapshotCmd *n = makeNode(ReplicationSlotSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_REPLICATION_SLOT_SNAPSHOT	{ $$ = "replication_slot_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..1988a6203b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+REPLICATION_SLOT_SNAPSHOT	{ return K_REPLICATION_SLOT_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_REPLICATION_SLOT_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 2f3c964824..b3ae11b2c8 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -478,6 +478,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 015ae2995d..70d926f4f0 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void ReplicationSlotSnapshot(ReplicationSlotSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1280,7 +1281,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1332,6 +1333,91 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+ReplicationSlotSnapshot(ReplicationSlotSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the replication slot and don't yet accept feedback messages or send
+	 * keepalives. As we possibly need to wait for further WAL the walsender
+	 * would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consisten WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1860,6 +1946,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_ReplicationSlotSnapshotCmd:
+			{
+				cmdtag = "REPLICATION_SLOT_SNAPSHOT";
+				set_ps_display(cmdtag);
+				ReplicationSlotSnapshot((ReplicationSlotSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..44a4580671 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		REPLICATION_SLOT_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct ReplicationSlotSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} ReplicationSlotSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index decffe352d..bd11f9f31e 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -384,6 +384,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -401,6 +411,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -435,6 +446,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 23bafec5f7..209918c380 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2322,6 +2322,7 @@ ReplicationSlotCtlData
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+ReplicationSlotSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
-- 
2.25.1

From fcac0624791d965687d284898237a2721afec4ab Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 2/2] Reuse Logical Replication Background worker

This commit allows tablesync workers to move to another table that needs synchronization,
when they're done with the current table in tablesync phase of Logical Replication.

Before this commit, tablesync workers were capable of syncing only one
relation. A new worker, replication slot and origin were needed for each
relation in the subscription.

Now, tablesync workers are not only limited with one relation and can move to another relation and reuse existing
replication slots and origins

This reduces the overhead of launching/killing a new background worker for each relation.
By reusing tablesync workers, replication slots and origins created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot/origin improves tablesync speed significantly especially for empty or small tables.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  29 ++
 src/backend/catalog/pg_subscription.c         | 284 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 238 ++++++----
 .../replication/logical/applyparallelworker.c |   7 +-
 src/backend/replication/logical/launcher.c    |   9 +-
 src/backend/replication/logical/tablesync.c   | 434 +++++++++++++-----
 src/backend/replication/logical/worker.c      | 395 ++++++++++------
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  15 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  33 +-
 src/test/regress/expected/misc_sanity.out     |  30 +-
 12 files changed, 1113 insertions(+), 370 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index c1e4048054..e34c129f0f 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8002,6 +8002,17 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+       The last used ID for tablesync workers. This ID is used to
+       create replication slots. The last used ID needs to be stored
+       to make logical replication can safely proceed after any interruption.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8086,6 +8097,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>text</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>text</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a56ae311c3..98d627ff49 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -114,6 +114,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +213,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +280,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,6 +309,14 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -275,6 +329,58 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +411,48 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +464,134 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Get origin name of subscription table.
+ *
+ * Returns null if the subscription table does not have a origin.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = TextDatumGetCString(d);
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * Returns null if the subscription table does not have a replication slot.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = TextDatumGetCString(d);
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index baff00dd74..2a68648a2b 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -649,6 +649,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(1);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -658,7 +659,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 
 	recordDependencyOnOwner(SubscriptionRelationId, subid, owner);
 
-	ReplicationOriginNameForLogicalRep(subid, InvalidOid, originname, sizeof(originname));
+	ReplicationOriginNameForLogicalRep(subid, originname, sizeof(originname), false);
 	replorigin_create(originname);
 
 	/*
@@ -709,7 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -799,6 +800,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -876,7 +879,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -900,6 +903,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -926,7 +930,29 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker if exists store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -946,8 +972,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 					 * origin and by this time the origin might be already
 					 * removed. For these reasons, passing missing_ok = true.
 					 */
-					ReplicationOriginNameForLogicalRep(sub->oid, relid, originname,
-													   sizeof(originname));
+					ReplicationOriginNameForLogicalRep(sub->oid, originname,
+													   sizeof(originname), true);
 					replorigin_drop_by_name(originname, true, false);
 				}
 
@@ -960,31 +986,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1320,8 +1339,8 @@ AlterSubscription(ParseState *pstate, AlterSubscriptionStmt *stmt,
 					char		originname[NAMEDATALEN];
 					XLogRecPtr	remote_lsn;
 
-					ReplicationOriginNameForLogicalRep(subid, InvalidOid,
-													   originname, sizeof(originname));
+					ReplicationOriginNameForLogicalRep(subid, originname,
+													   sizeof(originname), false);
 					originid = replorigin_by_name(originname, false);
 					remote_lsn = replorigin_get_progress(originid, false);
 
@@ -1384,6 +1403,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1455,6 +1475,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1504,37 +1532,29 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missin_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1542,7 +1562,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	RemoveSubscriptionRel(subid, InvalidOid);
 
 	/* Remove the origin tracking if exists. */
-	ReplicationOriginNameForLogicalRep(subid, InvalidOid, originname, sizeof(originname));
+	ReplicationOriginNameForLogicalRep(subid, originname, sizeof(originname), false);
 	replorigin_drop_by_name(originname, true, false);
 
 	/*
@@ -1586,39 +1606,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1641,6 +1639,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("not tuple returned."));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+	\
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -1995,6 +2058,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2005,18 +2069,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 2e5914d5d9..8df3a12c19 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										0);
 
 	if (launched)
 	{
@@ -942,8 +943,8 @@ ParallelApplyWorkerMain(Datum main_arg)
 
 	/* Setup replication origin tracking. */
 	StartTransactionCommand();
-	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-									   originname, sizeof(originname));
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, originname,
+									   sizeof(originname), false);
 	originid = replorigin_by_name(originname, false);
 
 	/*
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index afb7acddaa..f817037479 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -276,7 +276,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -401,7 +401,11 @@ retry:
 	/* Prepare the worker slot. */
 	worker->launch_time = now;
 	worker->in_use = true;
+	worker->is_first_run = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -409,6 +413,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->move_to_next_rel = false;
 	worker->stream_fileset = NULL;
 	worker->apply_leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
@@ -1015,7 +1020,7 @@ ApplyLauncherMain(Datum main_arg)
 					wait_time = wal_retrieve_retry_interval;
 
 					logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
-											 sub->owner, InvalidOid, DSM_HANDLE_INVALID);
+											 sub->owner, InvalidOid, DSM_HANDLE_INVALID, 0);
 				}
 			}
 
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 38dfce7129..01925d4e99 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -126,12 +126,8 @@ static bool FetchTableStates(bool *started_tx);
 
 static StringInfo copybuf = NULL;
 
-/*
- * Exit routine for synchronization worker.
- */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +139,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the leader apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -284,6 +290,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -292,6 +302,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		TimeLineID	tli;
 		char		syncslotname[NAMEDATALEN] = {0};
 		char		originname[NAMEDATALEN] = {0};
+		bool		is_streaming_ended = false;
 
 		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
 		MyLogicalRepWorker->relstate_lsn = current_lsn;
@@ -308,40 +319,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication trackin origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -350,34 +350,125 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			is_streaming_ended = true;
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
-		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   originname,
+											   sizeof(originname),
+											   true);
+
+			/* Drop replication origin */
+			replorigin_drop_by_name(originname, true, false);
+		}
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid))));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
+		StartTransactionCommand();
 
 		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
+		 */
+		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+										   originname,
+										   sizeof(originname),
+										   true);
+
+		/*
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
+
+		foreach(lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+			/*
+			 * Pick the table for the next run if there is not another worker
+			 * already picked that table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->is_first_run = false;
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->move_to_next_rel = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* Cleanup before next run or ending the worker. */
+		if (!MyLogicalRepWorker->move_to_next_rel)
+		{
+			/*
+			 * It is important to give an error if we are unable to drop the
+			 * slot, otherwise, it won't be dropped till the corresponding
+			 * subscription is dropped. So passing missing_ok = false.
+			 */
+			if (MyLogicalRepWorker->created_slot)
+			{
+				/* End streaming if it's not already ended. */
+				if (!is_streaming_ended)
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
+
+			/* Drop replication origin before exiting. */
+			replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -464,6 +555,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -484,18 +576,27 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -584,12 +685,22 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1190,8 +1301,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1202,11 +1313,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1229,6 +1340,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1257,7 +1369,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1277,11 +1389,25 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc0(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
-									   MyLogicalRepWorker->relid,
 									   originname,
-									   sizeof(originname));
+									   sizeof(originname),
+									   true);
+
+	CommitTransactionCommand();
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
@@ -1296,10 +1422,48 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication drop might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for relation %u",
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1320,6 +1484,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		goto copy_table_done;
 	}
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1327,10 +1492,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1369,6 +1536,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1384,55 +1552,99 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
 	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
+	 *
 	 * Prevent cancel/die interrupts while creating slot here because it is
 	 * possible that before the server finishes this command, a concurrent
 	 * drop subscription happens which would complete without removing this
 	 * slot leading to a dangling slot on the server.
+	 *
 	 */
-	HOLD_INTERRUPTS();
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-	RESUME_INTERRUPTS();
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		HOLD_INTERRUPTS();
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		RESUME_INTERRUPTS();
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin wit the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		HOLD_INTERRUPTS();
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		RESUME_INTERRUPTS();
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot. This is WAL
+	 * logged for the purpose of recovery. Locks are to prevent the
+	 * replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1455,10 +1667,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 79cda39445..032f402aed 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -385,6 +385,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -443,18 +444,26 @@ get_worker_name(void)
  * Form the origin name for the subscription.
  *
  * This is a common function for tablesync and other workers. Tablesync workers
- * must pass a valid relid. Other callers must pass relid = InvalidOid.
+ * must pass is_tablesync true so that origin name includes slot id.
  *
  * Return the name in the supplied buffer.
  */
 void
-ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
-								   char *originname, Size szoriginname)
+ReplicationOriginNameForLogicalRep(Oid suboid, char *originname,
+								   Size szoriginname, bool is_tablesync)
 {
-	if (OidIsValid(relid))
+	if (is_tablesync)
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, MyLogicalRepWorker->relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -3573,6 +3582,23 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can move to next table while running
+				 * process_syncing_tables_for_sync. Before we were able to
+				 * reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of moving to next table. Now that tablesync workers
+				 * can be reused, we need to take care of memory contexts here
+				 * before moving to sync a table.
+				 */
+				if (MyLogicalRepWorker->move_to_next_rel)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3592,6 +3618,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->move_to_next_rel)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -3694,8 +3724,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's moving to next relation, this is a sync worker. Sync workers
+	 * end the streaming during process_syncing_tables_for_sync. Calling
+	 * endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->move_to_next_rel)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -4277,6 +4315,56 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* stream_build_options
+  * 	Build logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4351,6 +4439,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4388,6 +4479,135 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->move_to_next_rel = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   originname,
+									   originname_size,
+									   true);
+	CommitTransactionCommand();
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	/* This is the leader apply worker */
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+
+	slotname = MySubscription->slotname;
+
+	/*
+		* This shouldn't happen if the subscription is enabled, but guard
+		* against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+		* crash if slot is NULL.)
+		*/
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, originname,
+										originname_size, false);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+		* We don't really use the output identify_system for anything but it
+		* does some initializations on the upstream so let's still call it.
+		*/
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+}
+
 /*
  * Common initialization for leader apply worker and parallel apply worker.
  *
@@ -4477,7 +4697,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -4505,156 +4724,48 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * The loop where worker does its job. It loops until the worker is not
+	 * reused.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
-
-	if (!am_tablesync_worker())
+	while (MyLogicalRepWorker->is_first_run ||
+		   MyLogicalRepWorker->move_to_next_rel)
 	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		if (am_tablesync_worker())
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * This is a tablesync worker. Start syncing tables before
+			 * starting the apply loop.
+			 */
+			run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 		else
 		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* This is main apply worker */
+			run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+		/* Run the main loop. */
+		start_apply(origin_startpos);
 
+		if (MyLogicalRepWorker->move_to_next_rel)
+		{
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
+			CommitTransactionCommand();
+		}
+	}
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index b0f2a1705d..a0ee12e259 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -103,6 +103,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -137,6 +140,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -157,6 +162,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..a35d04cccd 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	text		srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	text		srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,17 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8872c80cdf..3547daaaec 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -219,8 +219,9 @@ extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index db891eea8a..aee0d85f1d 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,26 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if worker is running for the first time or in reuse
+	 */
+	bool		is_first_run;
+
+	/*
+	 * Indicates if the sync worker created a replication slot or it reuses an
+	 * existing one created by another worker.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -56,6 +76,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Used to indicate whether sync worker will be reused for another
+	 * relation
+	 */
+	bool		move_to_next_rel;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -231,7 +257,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(int slot_no, uint16 generation);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
@@ -239,8 +266,8 @@ extern void logicalrep_worker_wakeup_ptr(LogicalRepWorker *worker);
 
 extern int	logicalrep_sync_worker_count(Oid subid);
 
-extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
-											   char *originname, Size szoriginname);
+extern void ReplicationOriginNameForLogicalRep(Oid suboid, char *originname,
+											   Size szoriginname, bool is_tablesync);
 extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos);
 
 extern bool AllTablesyncsReady(void);
diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out
index a57fd142a9..3d34a21421 100644
--- a/src/test/regress/expected/misc_sanity.out
+++ b/src/test/regress/expected/misc_sanity.out
@@ -47,20 +47,22 @@ WHERE c.oid < 16384 AND
       relkind = 'r' AND
       attstorage != 'p'
 ORDER BY 1, 2;
-         relname         |    attname    |   atttypid   
--------------------------+---------------+--------------
- pg_attribute            | attacl        | aclitem[]
- pg_attribute            | attfdwoptions | text[]
- pg_attribute            | attmissingval | anyarray
- pg_attribute            | attoptions    | text[]
- pg_class                | relacl        | aclitem[]
- pg_class                | reloptions    | text[]
- pg_class                | relpartbound  | pg_node_tree
- pg_index                | indexprs      | pg_node_tree
- pg_index                | indpred       | pg_node_tree
- pg_largeobject          | data          | bytea
- pg_largeobject_metadata | lomacl        | aclitem[]
-(11 rows)
+         relname         |     attname     |   atttypid   
+-------------------------+-----------------+--------------
+ pg_attribute            | attacl          | aclitem[]
+ pg_attribute            | attfdwoptions   | text[]
+ pg_attribute            | attmissingval   | anyarray
+ pg_attribute            | attoptions      | text[]
+ pg_class                | relacl          | aclitem[]
+ pg_class                | reloptions      | text[]
+ pg_class                | relpartbound    | pg_node_tree
+ pg_index                | indexprs        | pg_node_tree
+ pg_index                | indpred         | pg_node_tree
+ pg_largeobject          | data            | bytea
+ pg_largeobject_metadata | lomacl          | aclitem[]
+ pg_subscription_rel     | srreloriginname | text
+ pg_subscription_rel     | srrelslotname   | text
+(13 rows)
 
 -- system catalogs without primary keys
 --
-- 
2.25.1

From 1ce1967e48fd7a40a1344067ab14caa751b625fe Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 1/2] Add replication protocol cmd to create a snapshot

Introduced REPLICATION_SLOT_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

REPLICATION_SLOT_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

REPLICATION_SLOT_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy a relation and then apply changes by consuming from replication slot.
---
 doc/src/sgml/protocol.sgml                    | 32 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       | 69 ++++++++++++-
 src/backend/replication/logical/logical.c     | 39 +++++++-
 .../replication/logical/logicalfuncs.c        |  1 +
 src/backend/replication/repl_gram.y           | 18 +++-
 src/backend/replication/repl_scanner.l        |  2 +
 src/backend/replication/slotfuncs.c           |  1 +
 src/backend/replication/walsender.c           | 97 ++++++++++++++++++-
 src/include/nodes/replnodes.h                 | 11 +++
 src/include/replication/logical.h             |  1 +
 src/include/replication/walreceiver.h         | 13 +++
 src/tools/pgindent/typedefs.list              |  2 +
 12 files changed, 282 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 93fc7167d4..93a3867996 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2613,6 +2613,38 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>REPLICATION_SLOT_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>REPLICATION_SLOT_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the currenct transaction. This command is currently only supported
+       for logical replication.
+       slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index c40c6220db..9213fd2b1b 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -80,6 +80,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -96,7 +98,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -968,6 +971,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "REPLICATION_SLOT_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 1a58dd7649..d3bffd4cb2 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -461,6 +461,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, create a snapshot able to read all tables,
+ * 		otherwise do not create any snapshot.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -479,6 +483,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -487,6 +492,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -533,8 +539,39 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index fa1b641a2b..1191c70eb0 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..e5f0235d1e 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_REPLICATION_SLOT_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show replication_slot_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| replication_slot_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * REPLICATION_SLOT_SNAPSHOT %s options
+ */
+replication_slot_snapshot:
+			K_REPLICATION_SLOT_SNAPSHOT var_name plugin_options
+				{
+					ReplicationSlotSnapshotCmd *n = makeNode(ReplicationSlotSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_REPLICATION_SLOT_SNAPSHOT	{ $$ = "replication_slot_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..1988a6203b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+REPLICATION_SLOT_SNAPSHOT	{ return K_REPLICATION_SLOT_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_REPLICATION_SLOT_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 2f3c964824..b3ae11b2c8 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -478,6 +478,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 4ed3747e3f..7acfa88fad 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void ReplicationSlotSnapshot(ReplicationSlotSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1280,7 +1281,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1332,6 +1333,91 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+ReplicationSlotSnapshot(ReplicationSlotSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"REPLICATION_SLOT_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the replication slot and don't yet accept feedback messages or send
+	 * keepalives. As we possibly need to wait for further WAL the walsender
+	 * would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consisten WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1860,6 +1946,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_ReplicationSlotSnapshotCmd:
+			{
+				cmdtag = "REPLICATION_SLOT_SNAPSHOT";
+				set_ps_display(cmdtag);
+				ReplicationSlotSnapshot((ReplicationSlotSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..44a4580671 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		REPLICATION_SLOT_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct ReplicationSlotSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} ReplicationSlotSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index decffe352d..bd11f9f31e 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -384,6 +384,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -401,6 +411,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -435,6 +446,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 24510ac29e..fa38ea91e2 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2321,6 +2321,7 @@ ReplicationSlotCtlData
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+ReplicationSlotSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3794,6 +3795,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.25.1

From 6b26b1f21e410b2c649da401b33e8b1789a39ab7 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 2/2] Reuse Logical Replication Background worker

This commit allows tablesync workers to move to another table that needs synchronization,
when they're done with the current table in tablesync phase of Logical Replication.

Before this commit, tablesync workers were capable of syncing only one
relation. A new worker, replication slot and origin were needed for each
relation in the subscription.

Now, tablesync workers are not only limited with one relation and can move to another relation and reuse existing
replication slots and origins

This reduces the overhead of launching/killing a new background worker for each relation.
By reusing tablesync workers, replication slots and origins created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot/origin improves tablesync speed significantly especially for empty or small tables.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  30 ++
 src/backend/catalog/pg_subscription.c         | 284 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 226 ++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   9 +-
 src/backend/replication/logical/tablesync.c   | 426 +++++++++++++-----
 src/backend/replication/logical/worker.c      | 388 ++++++++++------
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  15 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  29 +-
 src/test/regress/expected/misc_sanity.out     |  30 +-
 12 files changed, 1097 insertions(+), 352 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index c1e4048054..eb8d4e1005 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8002,6 +8002,18 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+       The last used ID for tablesync workers. This ID is used to
+       create replication slots. The last used ID needs to be stored
+       to make logical replication can safely proceed after any interruption.
+       If sublastusedid is 0, then no table has been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8086,6 +8098,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>text</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>text</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a56ae311c3..f8dcad16ee 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -114,6 +114,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +213,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +280,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,6 +309,14 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -275,6 +329,58 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +411,48 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +464,134 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = TextDatumGetCString(d);
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = TextDatumGetCString(d);
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 464db6d247..1c4033fc79 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -649,6 +649,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -709,7 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -799,6 +800,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -876,7 +879,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -900,6 +903,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -926,7 +930,29 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker if exists store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -960,31 +986,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1384,6 +1403,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1455,6 +1475,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1504,6 +1532,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1515,36 +1545,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missin_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1596,39 +1616,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1651,6 +1649,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("not tuple returned."));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+	\
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2005,6 +2068,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2015,18 +2079,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 3579e704fe..1cab625dfb 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										0);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 564bffe5ca..b21951bc8f 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -429,7 +429,11 @@ retry:
 	/* Prepare the worker slot. */
 	worker->launch_time = now;
 	worker->in_use = true;
+	worker->is_first_run = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -437,6 +441,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->move_to_next_rel = false;
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
@@ -1155,7 +1160,7 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID, 0);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 4647837b82..821351198d 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -126,12 +126,8 @@ static bool FetchTableStates(bool *started_tx);
 
 static StringInfo copybuf = NULL;
 
-/*
- * Exit routine for synchronization worker.
- */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +139,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the leader apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -284,6 +290,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -292,6 +302,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		TimeLineID	tli;
 		char		syncslotname[NAMEDATALEN] = {0};
 		char		originname[NAMEDATALEN] = {0};
+		bool		is_streaming_ended = false;
 
 		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
 		MyLogicalRepWorker->relstate_lsn = current_lsn;
@@ -308,40 +319,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication trackin origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -350,34 +350,125 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			is_streaming_ended = true;
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
 
+			/* Drop replication origin */
+			replorigin_drop_by_name(originname, true, false);
+		}
+
+		/*
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
+		 */
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid))));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
+		StartTransactionCommand();
+
+		/*
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
+		 */
 		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
 										   MyLogicalRepWorker->relid,
 										   originname,
 										   sizeof(originname));
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
-		 */
-		replorigin_drop_by_name(originname, true, false);
+		foreach(lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
 
-		finish_sync_worker();
+			/*
+			 * Pick the table for the next run if there is not another worker
+			 * already picked that table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->is_first_run = false;
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->move_to_next_rel = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* Cleanup before next run or ending the worker. */
+		if (!MyLogicalRepWorker->move_to_next_rel)
+		{
+			/*
+			 * It is important to give an error if we are unable to drop the
+			 * slot, otherwise, it won't be dropped till the corresponding
+			 * subscription is dropped. So passing missing_ok = false.
+			 */
+			if (MyLogicalRepWorker->created_slot)
+			{
+				/* End streaming if it's not already ended. */
+				if (!is_streaming_ended)
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
+
+			/* Drop replication origin before exiting. */
+			replorigin_drop_by_name(originname, true, false);
+
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -464,6 +555,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -484,18 +576,27 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -584,12 +685,22 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1198,8 +1309,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1210,11 +1321,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1237,6 +1348,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1265,7 +1377,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1285,12 +1397,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc0(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1304,10 +1430,48 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication drop might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for relation %u",
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1328,6 +1492,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		goto copy_table_done;
 	}
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1335,10 +1500,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1377,6 +1544,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1392,55 +1560,99 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
 	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
+	 *
 	 * Prevent cancel/die interrupts while creating slot here because it is
 	 * possible that before the server finishes this command, a concurrent
 	 * drop subscription happens which would complete without removing this
 	 * slot leading to a dangling slot on the server.
+	 *
 	 */
-	HOLD_INTERRUPTS();
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-	RESUME_INTERRUPTS();
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		HOLD_INTERRUPTS();
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		RESUME_INTERRUPTS();
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		HOLD_INTERRUPTS();
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		RESUME_INTERRUPTS();
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot. This is WAL
+	 * logged for the purpose of recovery. Locks are to prevent the
+	 * replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1463,10 +1675,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index cfb2ab6248..c12924e4da 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -388,6 +388,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -456,13 +457,22 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
 		/* Replication origin name for non-tablesync workers. */
 		snprintf(originname, szoriginname, "pg_%u", suboid);
+		elog(LOG, "apply worker originname %s", originname);
 	}
 }
 
@@ -3576,6 +3586,23 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can move to next table while running
+				 * process_syncing_tables_for_sync. Before we were able to
+				 * reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of moving to next table. Now that tablesync workers
+				 * can be reused, we need to take care of memory contexts here
+				 * before moving to sync a table.
+				 */
+				if (MyLogicalRepWorker->move_to_next_rel)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3595,6 +3622,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->move_to_next_rel)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -3697,8 +3728,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's moving to next relation, this is a sync worker. Sync workers
+	 * end the streaming during process_syncing_tables_for_sync. Calling
+	 * endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->move_to_next_rel)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -4282,6 +4321,56 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* stream_build_options
+  * 	Build logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4356,6 +4445,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4393,6 +4485,135 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->move_to_next_rel = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+	CommitTransactionCommand();
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	/* This is the leader apply worker */
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+
+	slotname = MySubscription->slotname;
+
+	/*
+		* This shouldn't happen if the subscription is enabled, but guard
+		* against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+		* crash if slot is NULL.)
+		*/
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+		* We don't really use the output identify_system for anything but it
+		* does some initializations on the upstream so let's still call it.
+		*/
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+}
+
 /*
  * Common initialization for leader apply worker and parallel apply worker.
  *
@@ -4485,7 +4706,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -4513,156 +4733,48 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * The loop where worker does its job. It loops until the worker is not
+	 * reused.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
-
-	if (!am_tablesync_worker())
+	while (MyLogicalRepWorker->is_first_run ||
+		   MyLogicalRepWorker->move_to_next_rel)
 	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		if (am_tablesync_worker())
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * This is a tablesync worker. Start syncing tables before
+			 * starting the apply loop.
+			 */
+			run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 		else
 		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* This is main apply worker */
+			run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+		/* Run the main loop. */
+		start_apply(origin_startpos);
 
+		if (MyLogicalRepWorker->move_to_next_rel)
+		{
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
+			CommitTransactionCommand();
+		}
+	}
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index b0f2a1705d..a0ee12e259 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -103,6 +103,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -137,6 +140,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -157,6 +162,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..a35d04cccd 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	text		srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	text		srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,17 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8872c80cdf..3547daaaec 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -219,8 +219,9 @@ extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index dc87a4edd1..5f4b7b1f7c 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,26 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if worker is running for the first time or in reuse
+	 */
+	bool		is_first_run;
+
+	/*
+	 * Indicates if the sync worker created a replication slot or it reuses an
+	 * existing one created by another worker.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -56,6 +76,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Used to indicate whether sync worker will be reused for another
+	 * relation
+	 */
+	bool		move_to_next_rel;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -231,7 +257,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(int slot_no, uint16 generation);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out
index a57fd142a9..3d34a21421 100644
--- a/src/test/regress/expected/misc_sanity.out
+++ b/src/test/regress/expected/misc_sanity.out
@@ -47,20 +47,22 @@ WHERE c.oid < 16384 AND
       relkind = 'r' AND
       attstorage != 'p'
 ORDER BY 1, 2;
-         relname         |    attname    |   atttypid   
--------------------------+---------------+--------------
- pg_attribute            | attacl        | aclitem[]
- pg_attribute            | attfdwoptions | text[]
- pg_attribute            | attmissingval | anyarray
- pg_attribute            | attoptions    | text[]
- pg_class                | relacl        | aclitem[]
- pg_class                | reloptions    | text[]
- pg_class                | relpartbound  | pg_node_tree
- pg_index                | indexprs      | pg_node_tree
- pg_index                | indpred       | pg_node_tree
- pg_largeobject          | data          | bytea
- pg_largeobject_metadata | lomacl        | aclitem[]
-(11 rows)
+         relname         |     attname     |   atttypid   
+-------------------------+-----------------+--------------
+ pg_attribute            | attacl          | aclitem[]
+ pg_attribute            | attfdwoptions   | text[]
+ pg_attribute            | attmissingval   | anyarray
+ pg_attribute            | attoptions      | text[]
+ pg_class                | relacl          | aclitem[]
+ pg_class                | reloptions      | text[]
+ pg_class                | relpartbound    | pg_node_tree
+ pg_index                | indexprs        | pg_node_tree
+ pg_index                | indpred         | pg_node_tree
+ pg_largeobject          | data            | bytea
+ pg_largeobject_metadata | lomacl          | aclitem[]
+ pg_subscription_rel     | srreloriginname | text
+ pg_subscription_rel     | srrelslotname   | text
+(13 rows)
 
 -- system catalogs without primary keys
 --
-- 
2.25.1

From 757e519b98805ace371e5c3f972979d2b85ffd0c Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 2/2] Reuse Logical Replication Background worker

This commit allows tablesync workers to move to another table that needs synchronization,
when they're done with the current table in tablesync phase of Logical Replication.

Before this commit, tablesync workers were capable of syncing only one
relation. A new worker, replication slot and origin were needed for each
relation in the subscription.

Now, tablesync workers are not only limited with one relation and can move to another relation and reuse existing
replication slots and origins

This reduces the overhead of launching/killing a new background worker for each relation.
By reusing tablesync workers, replication slots and origins created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot/origin improves tablesync speed significantly especially for empty or small tables.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  30 ++
 src/backend/catalog/pg_subscription.c         | 284 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 226 ++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   9 +-
 src/backend/replication/logical/tablesync.c   | 421 +++++++++++++-----
 src/backend/replication/logical/worker.c      | 388 ++++++++++------
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  15 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  29 +-
 src/test/regress/expected/misc_sanity.out     |  30 +-
 12 files changed, 1094 insertions(+), 350 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index c1e4048054..eb8d4e1005 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8002,6 +8002,18 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+       The last used ID for tablesync workers. This ID is used to
+       create replication slots. The last used ID needs to be stored
+       to make logical replication can safely proceed after any interruption.
+       If sublastusedid is 0, then no table has been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8086,6 +8098,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>text</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>text</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a56ae311c3..f8dcad16ee 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -114,6 +114,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +213,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +280,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,6 +309,14 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -275,6 +329,58 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +411,48 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +464,134 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = TextDatumGetCString(d);
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = TextDatumGetCString(d);
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 464db6d247..1c4033fc79 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -649,6 +649,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -709,7 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -799,6 +800,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -876,7 +879,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -900,6 +903,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -926,7 +930,29 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker if exists store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -960,31 +986,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1384,6 +1403,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1455,6 +1475,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1504,6 +1532,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1515,36 +1545,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missin_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1596,39 +1616,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1651,6 +1649,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("not tuple returned."));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+	\
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2005,6 +2068,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2015,18 +2079,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 3579e704fe..1cab625dfb 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										0);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 970d170e73..77738e94a3 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -429,7 +429,11 @@ retry:
 	/* Prepare the worker slot. */
 	worker->launch_time = now;
 	worker->in_use = true;
+	worker->is_first_run = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -437,6 +441,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->move_to_next_rel = false;
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
@@ -1155,7 +1160,7 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID, 0);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 07eea504ba..f39c037d79 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -126,12 +126,8 @@ static bool FetchTableStates(bool *started_tx);
 
 static StringInfo copybuf = NULL;
 
-/*
- * Exit routine for synchronization worker.
- */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +139,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the leader apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -284,6 +290,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -292,6 +302,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		TimeLineID	tli;
 		char		syncslotname[NAMEDATALEN] = {0};
 		char		originname[NAMEDATALEN] = {0};
+		bool		is_streaming_ended = false;
 
 		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
 		MyLogicalRepWorker->relstate_lsn = current_lsn;
@@ -308,40 +319,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication trackin origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -350,34 +350,125 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
+		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
+		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			is_streaming_ended = true;
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/* Drop replication origin */
+			replorigin_drop_by_name(originname, true, false);
+		}
+
+		/*
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid))));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
 		StartTransactionCommand();
 
+		/*
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
+		 */
 		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
 										   MyLogicalRepWorker->relid,
 										   originname,
 										   sizeof(originname));
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
-		 */
-		replorigin_drop_by_name(originname, true, false);
+		foreach(lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+			/*
+			 * Pick the table for the next run if there is not another worker
+			 * already picked that table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->is_first_run = false;
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->move_to_next_rel = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* Cleanup before next run or ending the worker. */
+		if (!MyLogicalRepWorker->move_to_next_rel)
+		{
+			/*
+			 * It is important to give an error if we are unable to drop the
+			 * slot, otherwise, it won't be dropped till the corresponding
+			 * subscription is dropped. So passing missing_ok = false.
+			 */
+			if (MyLogicalRepWorker->created_slot)
+			{
+				/* End streaming if it's not already ended. */
+				if (!is_streaming_ended)
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
 
-		finish_sync_worker();
+			/* Drop replication origin before exiting. */
+			replorigin_drop_by_name(originname, true, false);
+
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -464,6 +555,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -484,18 +576,27 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -584,12 +685,22 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1198,8 +1309,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1210,11 +1321,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1237,6 +1348,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1265,7 +1377,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1285,12 +1397,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc0(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1304,10 +1430,48 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication drop might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for relation %u",
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1328,6 +1492,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		goto copy_table_done;
 	}
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1335,10 +1500,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1377,6 +1544,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1392,48 +1560,91 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		HOLD_INTERRUPTS();
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		RESUME_INTERRUPTS();
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot. This is WAL
+	 * logged for the purpose of recovery. Locks are to prevent the
+	 * replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1456,10 +1667,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index cfb2ab6248..c12924e4da 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -388,6 +388,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -456,13 +457,22 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
 		/* Replication origin name for non-tablesync workers. */
 		snprintf(originname, szoriginname, "pg_%u", suboid);
+		elog(LOG, "apply worker originname %s", originname);
 	}
 }
 
@@ -3576,6 +3586,23 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can move to next table while running
+				 * process_syncing_tables_for_sync. Before we were able to
+				 * reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of moving to next table. Now that tablesync workers
+				 * can be reused, we need to take care of memory contexts here
+				 * before moving to sync a table.
+				 */
+				if (MyLogicalRepWorker->move_to_next_rel)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3595,6 +3622,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->move_to_next_rel)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -3697,8 +3728,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's moving to next relation, this is a sync worker. Sync workers
+	 * end the streaming during process_syncing_tables_for_sync. Calling
+	 * endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->move_to_next_rel)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -4282,6 +4321,56 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* stream_build_options
+  * 	Build logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4356,6 +4445,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4393,6 +4485,135 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->move_to_next_rel = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+	CommitTransactionCommand();
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	/* This is the leader apply worker */
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+
+	slotname = MySubscription->slotname;
+
+	/*
+		* This shouldn't happen if the subscription is enabled, but guard
+		* against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+		* crash if slot is NULL.)
+		*/
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+		* We don't really use the output identify_system for anything but it
+		* does some initializations on the upstream so let's still call it.
+		*/
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+}
+
 /*
  * Common initialization for leader apply worker and parallel apply worker.
  *
@@ -4485,7 +4706,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -4513,156 +4733,48 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * The loop where worker does its job. It loops until the worker is not
+	 * reused.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
-
-	if (!am_tablesync_worker())
+	while (MyLogicalRepWorker->is_first_run ||
+		   MyLogicalRepWorker->move_to_next_rel)
 	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		if (am_tablesync_worker())
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * This is a tablesync worker. Start syncing tables before
+			 * starting the apply loop.
+			 */
+			run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 		else
 		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* This is main apply worker */
+			run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+		/* Run the main loop. */
+		start_apply(origin_startpos);
 
+		if (MyLogicalRepWorker->move_to_next_rel)
+		{
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
+			CommitTransactionCommand();
+		}
+	}
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index b0f2a1705d..a0ee12e259 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -103,6 +103,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -137,6 +140,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -157,6 +162,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..a35d04cccd 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	text		srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	text		srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,17 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8872c80cdf..3547daaaec 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -219,8 +219,9 @@ extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index dc87a4edd1..5f4b7b1f7c 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,26 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if worker is running for the first time or in reuse
+	 */
+	bool		is_first_run;
+
+	/*
+	 * Indicates if the sync worker created a replication slot or it reuses an
+	 * existing one created by another worker.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -56,6 +76,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Used to indicate whether sync worker will be reused for another
+	 * relation
+	 */
+	bool		move_to_next_rel;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -231,7 +257,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(int slot_no, uint16 generation);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out
index a57fd142a9..3d34a21421 100644
--- a/src/test/regress/expected/misc_sanity.out
+++ b/src/test/regress/expected/misc_sanity.out
@@ -47,20 +47,22 @@ WHERE c.oid < 16384 AND
       relkind = 'r' AND
       attstorage != 'p'
 ORDER BY 1, 2;
-         relname         |    attname    |   atttypid   
--------------------------+---------------+--------------
- pg_attribute            | attacl        | aclitem[]
- pg_attribute            | attfdwoptions | text[]
- pg_attribute            | attmissingval | anyarray
- pg_attribute            | attoptions    | text[]
- pg_class                | relacl        | aclitem[]
- pg_class                | reloptions    | text[]
- pg_class                | relpartbound  | pg_node_tree
- pg_index                | indexprs      | pg_node_tree
- pg_index                | indpred       | pg_node_tree
- pg_largeobject          | data          | bytea
- pg_largeobject_metadata | lomacl        | aclitem[]
-(11 rows)
+         relname         |     attname     |   atttypid   
+-------------------------+-----------------+--------------
+ pg_attribute            | attacl          | aclitem[]
+ pg_attribute            | attfdwoptions   | text[]
+ pg_attribute            | attmissingval   | anyarray
+ pg_attribute            | attoptions      | text[]
+ pg_class                | relacl          | aclitem[]
+ pg_class                | reloptions      | text[]
+ pg_class                | relpartbound    | pg_node_tree
+ pg_index                | indexprs        | pg_node_tree
+ pg_index                | indpred         | pg_node_tree
+ pg_largeobject          | data            | bytea
+ pg_largeobject_metadata | lomacl          | aclitem[]
+ pg_subscription_rel     | srreloriginname | text
+ pg_subscription_rel     | srrelslotname   | text
+(13 rows)
 
 -- system catalogs without primary keys
 --
-- 
2.25.1

From 6c741266475dbe8f7088fa953b27ac6972987d18 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 1/2] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy a relation and then apply changes by consuming from replication slot.
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 +++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 102 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 287 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 93fc7167d4..948a5aabd4 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2613,6 +2613,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index 560ec974fa..9173163a07 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -81,6 +81,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -97,7 +99,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -933,6 +936,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 1a58dd7649..d1504f7ec6 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -461,6 +461,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, create a snapshot able to read all tables,
+ * 		otherwise do not create any snapshot.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -479,6 +483,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -487,6 +492,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -533,8 +539,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index fa1b641a2b..1191c70eb0 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 2f3c964824..b3ae11b2c8 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -478,6 +478,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 4ed3747e3f..09011d6add 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1280,7 +1281,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1332,6 +1333,96 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the replication slot and don't yet accept feedback messages or send
+	 * keepalives. As we possibly need to wait for further WAL the walsender
+	 * would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consisten WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1860,6 +1951,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index decffe352d..bd11f9f31e 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -384,6 +384,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -401,6 +411,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -435,6 +446,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 07fbb7ccf6..a3de441b7e 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2323,6 +2323,7 @@ ReplicationSlotCtlData
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3796,6 +3797,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.25.1

From 757e519b98805ace371e5c3f972979d2b85ffd0c Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 2/2] Reuse Logical Replication Background worker

This commit allows tablesync workers to move to another table that needs synchronization,
when they're done with the current table in tablesync phase of Logical Replication.

Before this commit, tablesync workers were capable of syncing only one
relation. A new worker, replication slot and origin were needed for each
relation in the subscription.

Now, tablesync workers are not only limited with one relation and can move to another relation and reuse existing
replication slots and origins

This reduces the overhead of launching/killing a new background worker for each relation.
By reusing tablesync workers, replication slots and origins created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot/origin improves tablesync speed significantly especially for empty or small tables.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  30 ++
 src/backend/catalog/pg_subscription.c         | 284 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 226 ++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   9 +-
 src/backend/replication/logical/tablesync.c   | 421 +++++++++++++-----
 src/backend/replication/logical/worker.c      | 388 ++++++++++------
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  15 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  29 +-
 src/test/regress/expected/misc_sanity.out     |  30 +-
 12 files changed, 1094 insertions(+), 350 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index c1e4048054..eb8d4e1005 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8002,6 +8002,18 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+       The last used ID for tablesync workers. This ID is used to
+       create replication slots. The last used ID needs to be stored
+       to make logical replication can safely proceed after any interruption.
+       If sublastusedid is 0, then no table has been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8086,6 +8098,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>text</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>text</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a56ae311c3..f8dcad16ee 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -114,6 +114,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +213,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +280,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,6 +309,14 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -275,6 +329,58 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = CStringGetTextDatum(relslotname);
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] = CStringGetTextDatum(reloriginname);
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +411,48 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +464,134 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = TextDatumGetCString(d);
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = TextDatumGetCString(d);
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 464db6d247..1c4033fc79 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -649,6 +649,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -709,7 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -799,6 +800,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -876,7 +879,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -900,6 +903,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -926,7 +930,29 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker if exists store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -960,31 +986,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1384,6 +1403,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1455,6 +1475,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1504,6 +1532,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1515,36 +1545,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missin_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1596,39 +1616,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1651,6 +1649,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("not tuple returned."));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+	\
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2005,6 +2068,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2015,18 +2079,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 3579e704fe..1cab625dfb 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										0);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 970d170e73..77738e94a3 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -429,7 +429,11 @@ retry:
 	/* Prepare the worker slot. */
 	worker->launch_time = now;
 	worker->in_use = true;
+	worker->is_first_run = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -437,6 +441,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->move_to_next_rel = false;
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
@@ -1155,7 +1160,7 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID, 0);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 07eea504ba..f39c037d79 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -126,12 +126,8 @@ static bool FetchTableStates(bool *started_tx);
 
 static StringInfo copybuf = NULL;
 
-/*
- * Exit routine for synchronization worker.
- */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +139,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the leader apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -284,6 +290,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -292,6 +302,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		TimeLineID	tli;
 		char		syncslotname[NAMEDATALEN] = {0};
 		char		originname[NAMEDATALEN] = {0};
+		bool		is_streaming_ended = false;
 
 		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
 		MyLogicalRepWorker->relstate_lsn = current_lsn;
@@ -308,40 +319,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication trackin origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -350,34 +350,125 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
+		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
+		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			is_streaming_ended = true;
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/* Drop replication origin */
+			replorigin_drop_by_name(originname, true, false);
+		}
+
+		/*
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid))));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
 		StartTransactionCommand();
 
+		/*
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
+		 */
 		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
 										   MyLogicalRepWorker->relid,
 										   originname,
 										   sizeof(originname));
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
-		 */
-		replorigin_drop_by_name(originname, true, false);
+		foreach(lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+			/*
+			 * Pick the table for the next run if there is not another worker
+			 * already picked that table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->is_first_run = false;
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->move_to_next_rel = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* Cleanup before next run or ending the worker. */
+		if (!MyLogicalRepWorker->move_to_next_rel)
+		{
+			/*
+			 * It is important to give an error if we are unable to drop the
+			 * slot, otherwise, it won't be dropped till the corresponding
+			 * subscription is dropped. So passing missing_ok = false.
+			 */
+			if (MyLogicalRepWorker->created_slot)
+			{
+				/* End streaming if it's not already ended. */
+				if (!is_streaming_ended)
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
 
-		finish_sync_worker();
+			/* Drop replication origin before exiting. */
+			replorigin_drop_by_name(originname, true, false);
+
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -464,6 +555,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -484,18 +576,27 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -584,12 +685,22 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1198,8 +1309,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1210,11 +1321,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1237,6 +1348,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1265,7 +1377,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1285,12 +1397,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc0(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1304,10 +1430,48 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication drop might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for relation %u",
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1328,6 +1492,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		goto copy_table_done;
 	}
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1335,10 +1500,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1377,6 +1544,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1392,48 +1560,91 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		HOLD_INTERRUPTS();
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		RESUME_INTERRUPTS();
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot. This is WAL
+	 * logged for the purpose of recovery. Locks are to prevent the
+	 * replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1456,10 +1667,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index cfb2ab6248..c12924e4da 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -388,6 +388,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -456,13 +457,22 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
 		/* Replication origin name for non-tablesync workers. */
 		snprintf(originname, szoriginname, "pg_%u", suboid);
+		elog(LOG, "apply worker originname %s", originname);
 	}
 }
 
@@ -3576,6 +3586,23 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can move to next table while running
+				 * process_syncing_tables_for_sync. Before we were able to
+				 * reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of moving to next table. Now that tablesync workers
+				 * can be reused, we need to take care of memory contexts here
+				 * before moving to sync a table.
+				 */
+				if (MyLogicalRepWorker->move_to_next_rel)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3595,6 +3622,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->move_to_next_rel)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -3697,8 +3728,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's moving to next relation, this is a sync worker. Sync workers
+	 * end the streaming during process_syncing_tables_for_sync. Calling
+	 * endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->move_to_next_rel)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -4282,6 +4321,56 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* stream_build_options
+  * 	Build logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4356,6 +4445,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4393,6 +4485,135 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->move_to_next_rel = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+	CommitTransactionCommand();
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	/* This is the leader apply worker */
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+
+	slotname = MySubscription->slotname;
+
+	/*
+		* This shouldn't happen if the subscription is enabled, but guard
+		* against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+		* crash if slot is NULL.)
+		*/
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+		* We don't really use the output identify_system for anything but it
+		* does some initializations on the upstream so let's still call it.
+		*/
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+}
+
 /*
  * Common initialization for leader apply worker and parallel apply worker.
  *
@@ -4485,7 +4706,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -4513,156 +4733,48 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * The loop where worker does its job. It loops until the worker is not
+	 * reused.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
-
-	if (!am_tablesync_worker())
+	while (MyLogicalRepWorker->is_first_run ||
+		   MyLogicalRepWorker->move_to_next_rel)
 	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		if (am_tablesync_worker())
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * This is a tablesync worker. Start syncing tables before
+			 * starting the apply loop.
+			 */
+			run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 		else
 		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* This is main apply worker */
+			run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+		/* Run the main loop. */
+		start_apply(origin_startpos);
 
+		if (MyLogicalRepWorker->move_to_next_rel)
+		{
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
+			CommitTransactionCommand();
+		}
+	}
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index b0f2a1705d..a0ee12e259 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -103,6 +103,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -137,6 +140,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -157,6 +162,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..a35d04cccd 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	text		srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	text		srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,17 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8872c80cdf..3547daaaec 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -219,8 +219,9 @@ extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index dc87a4edd1..5f4b7b1f7c 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,26 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if worker is running for the first time or in reuse
+	 */
+	bool		is_first_run;
+
+	/*
+	 * Indicates if the sync worker created a replication slot or it reuses an
+	 * existing one created by another worker.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -56,6 +76,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Used to indicate whether sync worker will be reused for another
+	 * relation
+	 */
+	bool		move_to_next_rel;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -231,7 +257,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(int slot_no, uint16 generation);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
diff --git a/src/test/regress/expected/misc_sanity.out b/src/test/regress/expected/misc_sanity.out
index a57fd142a9..3d34a21421 100644
--- a/src/test/regress/expected/misc_sanity.out
+++ b/src/test/regress/expected/misc_sanity.out
@@ -47,20 +47,22 @@ WHERE c.oid < 16384 AND
       relkind = 'r' AND
       attstorage != 'p'
 ORDER BY 1, 2;
-         relname         |    attname    |   atttypid   
--------------------------+---------------+--------------
- pg_attribute            | attacl        | aclitem[]
- pg_attribute            | attfdwoptions | text[]
- pg_attribute            | attmissingval | anyarray
- pg_attribute            | attoptions    | text[]
- pg_class                | relacl        | aclitem[]
- pg_class                | reloptions    | text[]
- pg_class                | relpartbound  | pg_node_tree
- pg_index                | indexprs      | pg_node_tree
- pg_index                | indpred       | pg_node_tree
- pg_largeobject          | data          | bytea
- pg_largeobject_metadata | lomacl        | aclitem[]
-(11 rows)
+         relname         |     attname     |   atttypid   
+-------------------------+-----------------+--------------
+ pg_attribute            | attacl          | aclitem[]
+ pg_attribute            | attfdwoptions   | text[]
+ pg_attribute            | attmissingval   | anyarray
+ pg_attribute            | attoptions      | text[]
+ pg_class                | relacl          | aclitem[]
+ pg_class                | reloptions      | text[]
+ pg_class                | relpartbound    | pg_node_tree
+ pg_index                | indexprs        | pg_node_tree
+ pg_index                | indpred         | pg_node_tree
+ pg_largeobject          | data            | bytea
+ pg_largeobject_metadata | lomacl          | aclitem[]
+ pg_subscription_rel     | srreloriginname | text
+ pg_subscription_rel     | srrelslotname   | text
+(13 rows)
 
 -- system catalogs without primary keys
 --
-- 
2.25.1

From 6b9a115dd35ee2e728abf8ee0a61e06c78a867b3 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 1/2] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy a relation and then apply changes by consuming from replication slot.
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 +++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 102 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 287 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 93fc7167d4..948a5aabd4 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2613,6 +2613,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index 560ec974fa..9173163a07 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -81,6 +81,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -97,7 +99,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -933,6 +936,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 1a58dd7649..d1504f7ec6 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -461,6 +461,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, create a snapshot able to read all tables,
+ * 		otherwise do not create any snapshot.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -479,6 +483,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -487,6 +492,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -533,8 +539,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index fa1b641a2b..1191c70eb0 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 2f3c964824..b3ae11b2c8 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -478,6 +478,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 4ed3747e3f..33e8c63719 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1280,7 +1281,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1332,6 +1333,96 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consisten WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1860,6 +1951,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index decffe352d..bd11f9f31e 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -384,6 +384,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -401,6 +411,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -435,6 +446,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 07fbb7ccf6..a3de441b7e 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2323,6 +2323,7 @@ ReplicationSlotCtlData
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3796,6 +3797,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.25.1

From 129c9a1ed564714539a6421f14ab6538444ca9d3 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 2/2] Reuse Logical Replication Background worker

This commit allows reusing tablesync workers for syncing more than one relation sequantially during their lifetime, instead of exiting after only syncing one relation.

Before this commit, tablesync workers were capable of syncing only one
relation. For each table, a sync worker was launched, then a new
replication slot and new origin were created.

Now, tablesync workers are not only limited with one relation and can move to another relation in the same subscription and reuse existing
replication slots and origins

This reduces the overhead of launching/killing a new background worker for each relation.
By reusing tablesync workers, replication slots and origins created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot/origin improves tablesync speed significantly especially for empty or small tables.

A new tablesync worker gets launched only if the number of tablesync
workers for the subscription does not exceed
max_sync_workers_per_subscription. If there is a table needs to be synced, a tablesync worker picks that up and processes it.The worker continues to picking new tables to sync until there is no table left for synchronization in the subscription.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create them. Otherwise the worker reuses
slot and origin created by itself earlier. Tables in FINISHEDCOPY are
expected to have a replication slot and origin. Tablesync worker proceed
with existing slot and origin of FINISHEDCOPY tables and do not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 288 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 226 ++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   8 +-
 src/backend/replication/logical/tablesync.c   | 424 +++++++++++++-----
 src/backend/replication/logical/worker.c      | 387 ++++++++++------
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  15 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  26 +-
 11 files changed, 1083 insertions(+), 334 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index c1e4048054..31b4cfbbe4 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8002,6 +8002,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8086,6 +8099,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a56ae311c3..29a8fc6919 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -114,6 +114,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +213,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +280,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,6 +309,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = 
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -275,6 +331,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +415,48 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +468,134 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 464db6d247..b580e59ffc 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -649,6 +649,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -709,7 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -799,6 +800,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -876,7 +879,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -900,6 +903,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -926,7 +930,29 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -960,31 +986,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1384,6 +1403,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1455,6 +1475,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1504,6 +1532,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1515,36 +1545,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1596,39 +1616,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1651,6 +1649,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+                    errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2005,6 +2068,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2015,18 +2079,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 3579e704fe..1cab625dfb 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										0);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 970d170e73..5ea88cedce 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -437,6 +440,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->ready_to_reuse = false;
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
@@ -1155,7 +1159,7 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID, 0);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 07eea504ba..ae49c729e7 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -127,11 +127,10 @@ static bool FetchTableStates(bool *started_tx);
 static StringInfo copybuf = NULL;
 
 /*
- * Exit routine for synchronization worker.
+ * Prepares the synchronization worker for reuse or exit.
  */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +142,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the leader apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -284,6 +293,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -292,6 +305,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		TimeLineID	tli;
 		char		syncslotname[NAMEDATALEN] = {0};
 		char		originname[NAMEDATALEN] = {0};
+		bool		is_streaming_ended = false;
 
 		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
 		MyLogicalRepWorker->relstate_lsn = current_lsn;
@@ -308,40 +322,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication trackin origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -350,34 +353,129 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			is_streaming_ended = true;
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/* Drop replication origin */
+			replorigin_drop_by_name(originname, true, false);
+		}
+
+		/*
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
+		 */
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid))));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
+		StartTransactionCommand();
+
+		/*
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
+		 */
 		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
 										   MyLogicalRepWorker->relid,
 										   originname,
 										   sizeof(originname));
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		foreach(lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+			/*
+			 * Pick the table for the next run if it is not already picked up
+			 * by another worker.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->ready_to_reuse = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* 
+		 * If the worker is ready to be reused, clean up the worker for next
+		 * relations. 
+		 * If there is no more work left for this worker, drop replication slot
+		 * and origin. Then stop the worker gracefully. 
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		if (!MyLogicalRepWorker->ready_to_reuse)
+		{
+			/*
+			 * It is important to give an error if we are unable to drop the
+			 * slot, otherwise, it won't be dropped till the corresponding
+			 * subscription is dropped. So passing missing_ok = false.
+			 */
+			if (MyLogicalRepWorker->created_slot)
+			{
+				/* End streaming if it's not already ended. */
+				if (!is_streaming_ended)
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
+
+			/* Drop replication origin before exiting. */
+			replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -464,6 +562,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -484,18 +583,27 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -584,12 +692,22 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1198,8 +1316,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1210,11 +1328,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1237,6 +1355,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1265,7 +1384,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1285,12 +1404,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1304,10 +1437,49 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1327,7 +1499,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1335,10 +1509,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1377,6 +1553,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1392,48 +1569,91 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		HOLD_INTERRUPTS();
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		RESUME_INTERRUPTS();
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1456,10 +1676,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index cfb2ab6248..fd39254ecc 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -388,6 +388,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -456,8 +457,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -3576,6 +3585,23 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can move to next table while running
+				 * process_syncing_tables_for_sync. Before we were able to
+				 * reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of moving to next table. Now that tablesync workers
+				 * can be reused, we need to take care of memory contexts here
+				 * before moving to sync a table.
+				 */
+				if (MyLogicalRepWorker->ready_to_reuse)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3595,6 +3621,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->ready_to_reuse)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -3697,8 +3727,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's moving to next relation, this is a sync worker. Sync workers
+	 * end the streaming during process_syncing_tables_for_sync. Calling
+	 * endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->ready_to_reuse)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -4282,6 +4320,56 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* stream_build_options
+  * 	Build logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4356,6 +4444,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4393,6 +4484,135 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->ready_to_reuse = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+	CommitTransactionCommand();
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	/* This is the leader apply worker */
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+
+	slotname = MySubscription->slotname;
+
+	/*
+		* This shouldn't happen if the subscription is enabled, but guard
+		* against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+		* crash if slot is NULL.)
+		*/
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+		* We don't really use the output identify_system for anything but it
+		* does some initializations on the upstream so let's still call it.
+		*/
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+}
+
 /*
  * Common initialization for leader apply worker and parallel apply worker.
  *
@@ -4485,7 +4705,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -4513,156 +4732,46 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * The loop where worker does its job. It loops until the worker is not
+	 * reused.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
-
-	if (!am_tablesync_worker())
-	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+	do {
+		if (am_tablesync_worker())
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * This is a tablesync worker. Start syncing tables before
+			 * starting the apply loop.
+			 */
+			run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 		else
 		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* This is main apply worker */
+			run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+		/* Run the main loop. */
+		start_apply(origin_startpos);
 
+		if (MyLogicalRepWorker->ready_to_reuse)
+		{
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
+			CommitTransactionCommand();
+		}
+	} while (MyLogicalRepWorker->ready_to_reuse);
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index b0f2a1705d..a0ee12e259 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -103,6 +103,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -137,6 +140,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -157,6 +162,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..a528866012 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,17 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8872c80cdf..3547daaaec 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -219,8 +219,9 @@ extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index dc87a4edd1..7f7cb75c81 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -56,6 +73,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Used to indicate whether sync worker will be reused for another
+	 * relation
+	 */
+	bool		ready_to_reuse;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -231,7 +254,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(int slot_no, uint16 generation);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
-- 
2.25.1

From 129c9a1ed564714539a6421f14ab6538444ca9d3 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 2/2] Reuse Logical Replication Background worker

This commit allows reusing tablesync workers for syncing more than one relation sequantially during their lifetime, instead of exiting after only syncing one relation.

Before this commit, tablesync workers were capable of syncing only one
relation. For each table, a sync worker was launched, then a new
replication slot and new origin were created.

Now, tablesync workers are not only limited with one relation and can move to another relation in the same subscription and reuse existing
replication slots and origins

This reduces the overhead of launching/killing a new background worker for each relation.
By reusing tablesync workers, replication slots and origins created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot/origin improves tablesync speed significantly especially for empty or small tables.

A new tablesync worker gets launched only if the number of tablesync
workers for the subscription does not exceed
max_sync_workers_per_subscription. If there is a table needs to be synced, a tablesync worker picks that up and processes it.The worker continues to picking new tables to sync until there is no table left for synchronization in the subscription.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create them. Otherwise the worker reuses
slot and origin created by itself earlier. Tables in FINISHEDCOPY are
expected to have a replication slot and origin. Tablesync worker proceed
with existing slot and origin of FINISHEDCOPY tables and do not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 288 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 226 ++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   8 +-
 src/backend/replication/logical/tablesync.c   | 424 +++++++++++++-----
 src/backend/replication/logical/worker.c      | 387 ++++++++++------
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  15 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  26 +-
 11 files changed, 1083 insertions(+), 334 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index c1e4048054..31b4cfbbe4 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8002,6 +8002,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8086,6 +8099,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a56ae311c3..29a8fc6919 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -114,6 +114,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +213,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +280,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,6 +309,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] = 
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -275,6 +331,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +415,48 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update the replication slot name of a subscription table.
+ */
+void
+UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +468,134 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
+
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 464db6d247..b580e59ffc 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -649,6 +649,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -709,7 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -799,6 +800,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -876,7 +879,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -900,6 +903,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -926,7 +930,29 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -960,31 +986,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1384,6 +1403,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1455,6 +1475,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1504,6 +1532,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1515,36 +1545,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1596,39 +1616,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1651,6 +1649,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+                    errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2005,6 +2068,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2015,18 +2079,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 3579e704fe..1cab625dfb 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										0);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 970d170e73..5ea88cedce 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -437,6 +440,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->ready_to_reuse = false;
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
@@ -1155,7 +1159,7 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID, 0);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 07eea504ba..ae49c729e7 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -127,11 +127,10 @@ static bool FetchTableStates(bool *started_tx);
 static StringInfo copybuf = NULL;
 
 /*
- * Exit routine for synchronization worker.
+ * Prepares the synchronization worker for reuse or exit.
  */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +142,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the leader apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -284,6 +293,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -292,6 +305,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		TimeLineID	tli;
 		char		syncslotname[NAMEDATALEN] = {0};
 		char		originname[NAMEDATALEN] = {0};
+		bool		is_streaming_ended = false;
 
 		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
 		MyLogicalRepWorker->relstate_lsn = current_lsn;
@@ -308,40 +322,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication trackin origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -350,34 +353,129 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			is_streaming_ended = true;
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/* Drop replication origin */
+			replorigin_drop_by_name(originname, true, false);
+		}
+
+		/*
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
+		 */
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid))));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
+		StartTransactionCommand();
+
+		/*
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
+		 */
 		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
 										   MyLogicalRepWorker->relid,
 										   originname,
 										   sizeof(originname));
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		foreach(lc, rstates)
+		{
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+			/*
+			 * Pick the table for the next run if it is not already picked up
+			 * by another worker.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->ready_to_reuse = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
+
+		/* 
+		 * If the worker is ready to be reused, clean up the worker for next
+		 * relations. 
+		 * If there is no more work left for this worker, drop replication slot
+		 * and origin. Then stop the worker gracefully. 
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		if (!MyLogicalRepWorker->ready_to_reuse)
+		{
+			/*
+			 * It is important to give an error if we are unable to drop the
+			 * slot, otherwise, it won't be dropped till the corresponding
+			 * subscription is dropped. So passing missing_ok = false.
+			 */
+			if (MyLogicalRepWorker->created_slot)
+			{
+				/* End streaming if it's not already ended. */
+				if (!is_streaming_ended)
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
+
+			/* Drop replication origin before exiting. */
+			replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -464,6 +562,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -484,18 +583,27 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -584,12 +692,22 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1198,8 +1316,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1210,11 +1328,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1237,6 +1355,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1265,7 +1384,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1285,12 +1404,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1304,10 +1437,49 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		slotname = prev_slotname;
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1327,7 +1499,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1335,10 +1509,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
 	pgstat_report_stat(true);
 
@@ -1377,6 +1553,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1392,48 +1569,91 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		HOLD_INTERRUPTS();
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		RESUME_INTERRUPTS();
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1456,10 +1676,12 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index cfb2ab6248..fd39254ecc 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -388,6 +388,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -456,8 +457,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -3576,6 +3585,23 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can move to next table while running
+				 * process_syncing_tables_for_sync. Before we were able to
+				 * reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of moving to next table. Now that tablesync workers
+				 * can be reused, we need to take care of memory contexts here
+				 * before moving to sync a table.
+				 */
+				if (MyLogicalRepWorker->ready_to_reuse)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3595,6 +3621,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->ready_to_reuse)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -3697,8 +3727,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's moving to next relation, this is a sync worker. Sync workers
+	 * end the streaming during process_syncing_tables_for_sync. Calling
+	 * endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->ready_to_reuse)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -4282,6 +4320,56 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* stream_build_options
+  * 	Build logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4356,6 +4444,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4393,6 +4484,135 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->ready_to_reuse = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+	CommitTransactionCommand();
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	/* This is the leader apply worker */
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+
+	slotname = MySubscription->slotname;
+
+	/*
+		* This shouldn't happen if the subscription is enabled, but guard
+		* against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+		* crash if slot is NULL.)
+		*/
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+		* We don't really use the output identify_system for anything but it
+		* does some initializations on the upstream so let's still call it.
+		*/
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+}
+
 /*
  * Common initialization for leader apply worker and parallel apply worker.
  *
@@ -4485,7 +4705,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -4513,156 +4732,46 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * The loop where worker does its job. It loops until the worker is not
+	 * reused.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
-
-	if (!am_tablesync_worker())
-	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+	do {
+		if (am_tablesync_worker())
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * This is a tablesync worker. Start syncing tables before
+			 * starting the apply loop.
+			 */
+			run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 		else
 		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* This is main apply worker */
+			run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+		/* Run the main loop. */
+		start_apply(origin_startpos);
 
+		if (MyLogicalRepWorker->ready_to_reuse)
+		{
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\".",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid))));
+			CommitTransactionCommand();
+		}
+	} while (MyLogicalRepWorker->ready_to_reuse);
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index b0f2a1705d..a0ee12e259 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -103,6 +103,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -137,6 +140,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -157,6 +162,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..a528866012 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,17 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+extern void UpdateSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *relslotname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8872c80cdf..3547daaaec 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -219,8 +219,9 @@ extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index dc87a4edd1..7f7cb75c81 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -56,6 +73,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Used to indicate whether sync worker will be reused for another
+	 * relation
+	 */
+	bool		ready_to_reuse;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -231,7 +254,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(int slot_no, uint16 generation);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
-- 
2.25.1

From 2959b1ae74318e6b110050b452ca06ad87f9cb6a Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 1/2] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy a relation and then apply changes by consuming from replication slot.
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 +++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 102 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 287 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 93fc7167d4..948a5aabd4 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2613,6 +2613,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index 560ec974fa..9173163a07 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -81,6 +81,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -97,7 +99,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -933,6 +936,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 1a58dd7649..d1504f7ec6 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -461,6 +461,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, create a snapshot able to read all tables,
+ * 		otherwise do not create any snapshot.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -479,6 +483,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -487,6 +492,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -533,8 +539,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index fa1b641a2b..1191c70eb0 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 2f3c964824..b3ae11b2c8 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -478,6 +478,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 4ed3747e3f..b10ae3eada 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1280,7 +1281,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1332,6 +1333,96 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consistent WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1860,6 +1951,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index decffe352d..bd11f9f31e 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -384,6 +384,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -401,6 +411,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -435,6 +446,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 07fbb7ccf6..a3de441b7e 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2323,6 +2323,7 @@ ReplicationSlotCtlData
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3796,6 +3797,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.25.1

From 1cb2c7e2e3efa1abd10f90e38b27220af8bcabf2 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH v8 1/2] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy a relation and then apply changes by consuming from replication slot.
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 ++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 104 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 289 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 93fc7167d4..948a5aabd4 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2613,6 +2613,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index 560ec974fa..9173163a07 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -81,6 +81,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -97,7 +99,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -933,6 +936,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index c3ec97a0a6..93fc0e5143 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -472,6 +472,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, must obtain a snapshot able to read all tables;
+ *  	if false, one that can read only catalogs is acceptable.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -490,6 +494,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -498,6 +503,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -544,8 +550,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index fa1b641a2b..1191c70eb0 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 2f3c964824..b3ae11b2c8 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -478,6 +478,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 75e8363e24..d2ae504ce5 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1280,7 +1281,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1332,6 +1333,98 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	CheckLogicalDecodingRequirements();
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consistent WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1860,6 +1953,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index decffe352d..bd11f9f31e 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -384,6 +384,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -401,6 +411,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -435,6 +446,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 22ea42c16b..fb5cc1d599 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2331,6 +2331,7 @@ ReplicationSlotCtlData
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3805,6 +3806,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.25.1

From bcfa6abff4feaa9f965bdf31cf6cedf8e53e285d Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH v11 2/2] Reuse Logical Replication Background worker

This commit allows reusing tablesync workers for syncing more than one relation sequantially during their lifetime, instead of exiting after only syncing one relation.

Before this commit, tablesync workers were capable of syncing only one
relation. For each table, a sync worker was launched, then a new
replication slot and new origin were created.

Now, tablesync workers are not only limited with one relation and can move to another relation in the same subscription and reuse existing
replication slots and origins

This reduces the overhead of launching/killing a new background worker for each relation.
By reusing tablesync workers, replication slots and origins created for tablesync can be reused as well.
Removing the burden of creating/dropping replication slot/origin improves tablesync speed significantly especially for empty or small tables.

A new tablesync worker gets launched only if the number of tablesync
workers for the subscription does not exceed
max_sync_workers_per_subscription. If there is a table needs to be synced, a tablesync worker picks that up and processes it.The worker continues to picking new tables to sync until there is no table left for synchronization in the subscription.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create them. Otherwise the worker reuses
slot and origin created by itself earlier. Tables in FINISHEDCOPY are
expected to have a replication slot and origin. Tablesync worker proceed
with existing slot and origin of FINISHEDCOPY tables and do not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 247 ++++++++-
 src/backend/commands/subscriptioncmds.c       | 226 ++++++---
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   9 +-
 src/backend/replication/logical/tablesync.c   | 478 ++++++++++++++----
 src/backend/replication/logical/worker.c      | 396 +++++++++------
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  14 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  29 +-
 11 files changed, 1107 insertions(+), 335 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index c1e4048054..31b4cfbbe4 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8002,6 +8002,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8086,6 +8099,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index a56ae311c3..cb4001825f 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -114,6 +114,14 @@ GetSubscription(Oid subid, bool missing_ok)
 	Assert(!isnull);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -205,6 +213,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -234,7 +280,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -263,6 +309,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -275,6 +331,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -305,14 +415,56 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -324,6 +476,85 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 464db6d247..49d5ec8eca 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -649,6 +649,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -709,7 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -799,6 +800,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	} SubRemoveRels;
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -876,7 +879,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -900,6 +903,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -926,7 +930,29 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
@@ -960,31 +986,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1384,6 +1403,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1455,6 +1475,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1504,6 +1532,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1515,36 +1545,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1596,39 +1616,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1651,6 +1649,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2005,6 +2068,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2015,18 +2079,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 4518683779..6114970289 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										InvalidRepSlotId);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 970d170e73..226aa3f58f 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -437,6 +440,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->ready_to_reuse = false;
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
@@ -1155,7 +1159,8 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID,
+										 InvalidRepSlotId);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 07eea504ba..6983871e6b 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -127,11 +127,10 @@ static bool FetchTableStates(bool *started_tx);
 static StringInfo copybuf = NULL;
 
 /*
- * Exit routine for synchronization worker.
+ * Prepares the synchronization worker for reuse or exit.
  */
 static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -143,18 +142,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the leader apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+static void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -284,6 +293,10 @@ invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue)
 static void
 process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 {
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
+
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_CATCHUP &&
@@ -292,6 +305,7 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		TimeLineID	tli;
 		char		syncslotname[NAMEDATALEN] = {0};
 		char		originname[NAMEDATALEN] = {0};
+		bool		is_streaming_ended = false;
 
 		MyLogicalRepWorker->relstate = SUBREL_STATE_SYNCDONE;
 		MyLogicalRepWorker->relstate_lsn = current_lsn;
@@ -308,40 +322,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication tracking origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -350,34 +353,148 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
+		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
+		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			is_streaming_ended = true;
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/*
+			 * Drop replication origin
+			 *
+			 * There is a chance that the user is concurrently performing refresh
+			 * for the subscription where we remove the table state and its origin
+			 * or the apply worker would have removed this origin. So passing
+			 * missing_ok = true.
+			 */
+			replorigin_drop_by_name(originname, true, false);
+		}
+
+		/*
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+		ereport(DEBUG2,
+			(errmsg("process_syncing_tables_for_sync: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					"NULL", "NULL", MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
 		StartTransactionCommand();
 
+		/*
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
+		 */
 		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
 										   MyLogicalRepWorker->relid,
 										   originname,
 										   sizeof(originname));
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
+		rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+
+		foreach(lc, rstates)
+		{
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+			/*
+			 * Pick the table for the next run if it is not already picked up
+			 * by another worker.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				MyLogicalRepWorker->ready_to_reuse = true;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
+		}
 
 		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		 * If the worker is ready to be reused, clean up the worker for next
+		 * relations.
+		 * If there is no more work left for this worker, drop replication slot
+		 * and origin. Then stop the worker gracefully.
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		if (!MyLogicalRepWorker->ready_to_reuse)
+		{
+			/*
+			 * It is important to give an error if we are unable to drop the
+			 * slot, otherwise, it won't be dropped till the corresponding
+			 * subscription is dropped. So passing missing_ok = false.
+			 */
+			if (MyLogicalRepWorker->created_slot)
+			{
+				/* End streaming if it's not already ended. */
+				if (!is_streaming_ended)
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+				ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
+			}
+
+			/*
+			 * Drop replication origin before exiting.
+			 *
+			 * There is a chance that the user is concurrently performing refresh
+			 * for the subscription where we remove the table state and its origin
+			 * or the apply worker would have removed this origin. So passing
+			 * missing_ok = true.
+			 */
+			replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+			finish_sync_worker();
+		}
+		else
+		{
+			clean_sync_worker();
+		}
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -464,6 +581,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -484,18 +602,31 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+				ereport(DEBUG2,
+					(errmsg("process_syncing_tables_for_apply: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+							"NULL", "NULL", rstate->state,
+							rstate->relid, MyLogicalRepWorker->subid)));
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -584,12 +715,25 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+						ereport(DEBUG2,
+								(errmsg("process_syncing_tables_for_apply: incremented lastusedid to %lld for subscription %u",
+										(long long) MySubscription->lastusedid, MySubscription->oid)));
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1198,8 +1342,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1210,11 +1354,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1237,6 +1381,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	WalRcvExecResult *res;
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1265,7 +1410,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1285,12 +1430,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1304,10 +1463,53 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+			ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL", "NULL", MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		memcpy(slotname, prev_slotname, NAMEDATALEN);
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1327,7 +1529,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1335,11 +1539,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+
+	/*
+	 * Refresh the originname in case of having non-existing origin
+	 * from previous failed sync attempts.
+	 * If that's the case, it should be removed from the catalog so far.
+	 * Then, we can continue by reusing the origin created by the current
+	 * worker instead of .
+	 */
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									MyLogicalRepWorker->relid,
+									originname,
+									sizeof(originname));
+
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					slotname, originname, MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
 	pgstat_report_stat(true);
 
 	StartTransactionCommand();
@@ -1377,6 +1601,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1392,48 +1617,95 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: created replication slot %s for subscription %u",
+						slotname, MyLogicalRepWorker->subid)));
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: reusing replication slot %s for relation %u in subscription %u",
+						slotname, MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1456,12 +1728,18 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					originname, slotname, SUBREL_STATE_FINISHEDCOPY,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 
 copy_table_done:
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index cfb2ab6248..9651d6f752 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -388,6 +388,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -456,8 +457,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -3576,6 +3585,27 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can move to next table while running
+				 * process_syncing_tables_for_sync. Before we were able to
+				 * reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of moving to next table. Now that tablesync workers
+				 * can be reused, we need to take care of memory contexts here
+				 * before moving to sync a table.
+				 */
+				if (MyLogicalRepWorker->ready_to_reuse)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+
+					/* Pop the error context stack */
+					error_context_stack = errcallback.previous;
+					apply_error_context_stack = error_context_stack;
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3595,6 +3625,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->ready_to_reuse)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -3697,8 +3731,16 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's moving to next relation, this is a sync worker. Sync workers
+	 * end the streaming during process_syncing_tables_for_sync. Calling
+	 * endstreaming twice causes "no COPY in progress" errors.
+	 */
+	if (!MyLogicalRepWorker->ready_to_reuse)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -4282,6 +4324,56 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* stream_build_options
+  * 	Build logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4356,6 +4448,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4393,6 +4488,135 @@ start_apply(XLogRecPtr origin_startpos)
 	PG_END_TRY();
 }
 
+/*
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->ready_to_reuse = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+	CommitTransactionCommand();
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	/* This is the leader apply worker */
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+
+	slotname = MySubscription->slotname;
+
+	/*
+		* This shouldn't happen if the subscription is enabled, but guard
+		* against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+		* crash if slot is NULL.)
+		*/
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+		* We don't really use the output identify_system for anything but it
+		* does some initializations on the upstream so let's still call it.
+		*/
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+}
+
 /*
  * Common initialization for leader apply worker and parallel apply worker.
  *
@@ -4464,9 +4688,10 @@ InitializeApplyWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started",
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has started",
 						MySubscription->name,
-						get_rel_name(MyLogicalRepWorker->relid))));
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
 	else
 		ereport(LOG,
 		/* translator: first %s is the name of logical replication worker */
@@ -4485,7 +4710,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	/* Attach to slot */
 	logicalrep_worker_attach(worker_slot);
@@ -4513,156 +4737,46 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * The loop where worker does its job. It loops until the worker is not
+	 * reused.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
-
-	if (!am_tablesync_worker())
-	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+	do {
+		if (am_tablesync_worker())
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * This is a tablesync worker. Start syncing tables before
+			 * starting the apply loop.
+			 */
+			run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 		else
 		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			/* This is leader apply worker */
+			run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 		}
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
-
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+		/* Run the main loop. */
+		start_apply(origin_startpos);
 
+		if (MyLogicalRepWorker->ready_to_reuse)
+		{
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid), MyLogicalRepWorker->relid)));
+			CommitTransactionCommand();
+		}
+	} while (MyLogicalRepWorker->ready_to_reuse);
 	proc_exit(0);
 }
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index b0f2a1705d..a0ee12e259 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -103,6 +103,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -137,6 +140,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -157,6 +162,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..185164d75e 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,16 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index 8872c80cdf..3547daaaec 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -219,8 +219,9 @@ extern bool InvalidateObsoleteReplicationSlots(XLogSegNo oldestSegno);
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index dc87a4edd1..353a066414 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -56,6 +73,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Used to indicate whether sync worker will be reused for another
+	 * relation
+	 */
+	bool		ready_to_reuse;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -231,7 +254,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(int slot_no, uint16 generation);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
@@ -324,4 +348,7 @@ am_parallel_apply_worker(void)
 	return isParallelApplyWorker(MyLogicalRepWorker);
 }
 
+/* Invalid identifier to be used for naming replication slots */
+#define InvalidRepSlotId	0
+
 #endif							/* WORKER_INTERNAL_H */
-- 
2.25.1

From 0c5bb1dacf97219970ea6c7802da5141393bcfa8 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Tue, 11 Apr 2023 14:24:22 +0300
Subject: [PATCH 1/3] Reuse Tablesync Workers

This commit allows reusing tablesync workers for syncing more than one
relation sequantially during their lifetime, instead of exiting after
only syncing one relation.

Before this commit, tablesync workers were capable of syncing only one
relation. For each table, a new sync worker was launched and killed
when the worker is done with the current table.

Now, tablesync workers are not only limited with one relation and can
move to another relation in the same subscription. This reduces the
overhead of launching/killing a new background worker for each relation.

A new tablesync worker gets launched only if the number of tablesync
workers for the subscription does not exceed
max_sync_workers_per_subscription. If there is a table needs to be synced,
a tablesync worker picks that up and syncs it.The worker continues to
picking new tables to sync until there is no table left for synchronization
in the subscription.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/postmaster/bgworker.c             |   3 +
 .../replication/logical/applyparallelworker.c |   2 +-
 src/backend/replication/logical/launcher.c    |   5 +-
 src/backend/replication/logical/tablesync.c   |  48 +-
 src/backend/replication/logical/worker.c      | 525 +++++++++++++-----
 src/include/replication/logicalworker.h       |   1 +
 src/include/replication/worker_internal.h     |  10 +-
 7 files changed, 411 insertions(+), 183 deletions(-)

diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 0dd22b2351..5609919edf 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -131,6 +131,9 @@ static const struct
 	},
 	{
 		"ParallelApplyWorkerMain", ParallelApplyWorkerMain
+	},
+	{
+		"TablesyncWorkerMain", TablesyncWorkerMain
 	}
 };
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index ee7a18137f..50aa1386d5 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -940,7 +940,7 @@ ParallelApplyWorkerMain(Datum main_arg)
 	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
 		MyLogicalRepWorker->reply_time = 0;
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index ceea126231..3e89366e7c 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -440,6 +440,7 @@ retry:
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
+	worker->ready_to_reuse = false;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
 	TIMESTAMP_NOBEGIN(worker->last_recv_time);
@@ -460,8 +461,10 @@ retry:
 
 	if (is_parallel_apply_worker)
 		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ParallelApplyWorkerMain");
-	else
+	else if (!OidIsValid(relid))
 		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
+	else
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "TablesyncWorkerMain");
 
 	if (OidIsValid(relid))
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 0c71ae9ba7..0a813bc371 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -127,40 +127,6 @@ static bool FetchTableStates(bool *started_tx);
 
 static StringInfo copybuf = NULL;
 
-/*
- * Exit routine for synchronization worker.
- */
-static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
-{
-	/*
-	 * Commit any outstanding transaction. This is the usual case, unless
-	 * there was nothing to do for the table.
-	 */
-	if (IsTransactionState())
-	{
-		CommitTransactionCommand();
-		pgstat_report_stat(true);
-	}
-
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
-
-	/* Find the leader apply worker and signal it. */
-	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
-
-	/* Stop gracefully */
-	proc_exit(0);
-}
-
 /*
  * Wait until the relation sync state is set in the catalog to the expected
  * one; return true when it happens.
@@ -378,10 +344,16 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 */
 		replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
-	}
-	else
+		/*
+		 * Sync worker is cleaned at this point. It's ready to sync next table,
+		 * if needed.
+		 */
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->ready_to_reuse = true;
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
+	}
+
+	SpinLockRelease(&MyLogicalRepWorker->relmutex);
 }
 
 /*
@@ -1275,7 +1247,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		case SUBREL_STATE_SYNCDONE:
 		case SUBREL_STATE_READY:
 		case SUBREL_STATE_UNKNOWN:
-			finish_sync_worker();	/* doesn't return */
+			sync_worker_exit();	/* doesn't return */
 	}
 
 	/* Calculate the name of the tablesync slot. */
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 879309b316..d5f87b84cb 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -392,6 +392,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -3617,6 +3618,27 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can go into process_syncing_tables_for_sync early.
+				 * Before we were able to reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of preparing for reuse. Now that tablesync workers
+				 * can be reused and process_syncing_tables_for_sync is not
+				 * responsible for exiting. We need to take care of memory
+				 * contexts here before moving to sync the nex table or exit.
+				 */
+				if (MyLogicalRepWorker->ready_to_reuse)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+
+					/* Pop the error context stack */
+					error_context_stack = errcallback.previous;
+					apply_error_context_stack = error_context_stack;
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3636,6 +3658,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->ready_to_reuse)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -3734,12 +3760,15 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 		}
 	}
 
-	/* Pop the error context stack */
-	error_context_stack = errcallback.previous;
-	apply_error_context_stack = error_context_stack;
-
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's still not ready to reuse, this is probably an apply worker.
+	 * End streaming before exiting.
+	 */
+	if (!MyLogicalRepWorker->ready_to_reuse)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -3865,6 +3894,50 @@ apply_worker_exit(void)
 	proc_exit(0);
 }
 
+/*
+ * Prepares the synchronization worker for reuse or exit.
+ */
+static void
+clean_sync_worker(void)
+{
+	/*
+	 * Commit any outstanding transaction. This is the usual case, unless
+	 * there was nothing to do for the table.
+	 */
+	if (IsTransactionState())
+	{
+		CommitTransactionCommand();
+		pgstat_report_stat(true);
+	}
+
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
+
+	/* Find the leader apply worker and signal it. */
+	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+void
+pg_attribute_noreturn()
+sync_worker_exit(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
+
+	/* Stop gracefully */
+	proc_exit(0);
+}
+
+
 /*
  * Reread subscription info if needed. Most changes will be exit.
  */
@@ -4324,6 +4397,56 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* stream_build_options
+  * 	Build logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4436,13 +4559,154 @@ start_apply(XLogRecPtr origin_startpos)
 }
 
 /*
- * Common initialization for leader apply worker and parallel apply worker.
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->ready_to_reuse = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+	CommitTransactionCommand();
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	/* Start applying changes to catcup. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	/* This is the leader apply worker */
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+	bool		must_use_password;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard
+	 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+	 * crash if slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	/* Is the use of a password mandatory? */
+	must_use_password = MySubscription->passwordrequired &&
+		!superuser_arg(MySubscription->owner);
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											must_use_password,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it
+	 * does some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+
+	/* Run the main loop. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Common initialization for logical replication workers; leader apply worker,
+ * parallel apply worker and tablesync worker.
  *
  * Initialize the database connection, in-memory subscription and necessary
  * config options.
  */
 void
-InitializeApplyWorker(void)
+InitializeLogRepWorker(void)
 {
 	MemoryContext oldctx;
 
@@ -4506,9 +4770,10 @@ InitializeApplyWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started",
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has started",
 						MySubscription->name,
-						get_rel_name(MyLogicalRepWorker->relid))));
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
 	else
 		ereport(LOG,
 		/* translator: first %s is the name of logical replication worker */
@@ -4527,7 +4792,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	InitializingApplyWorker = true;
 
@@ -4551,7 +4815,7 @@ ApplyWorkerMain(Datum main_arg)
 	/* Load the libpq-specific functions */
 	load_file("libpqwalreceiver", false);
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
@@ -4559,165 +4823,142 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
+	/*
+	 * Setup callback for syscache so that we know when something changes in
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
+	 */
+	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
+								  invalidate_syncing_table_states,
+								  (Datum) 0);
 
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-		bool		must_use_password;
+	/* This is leader apply worker */
+	run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 
-		myslotname = MySubscription->slotname;
+	proc_exit(0);
+}
 
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
+/* Logical Replication Tablesync worker entry point */
+void
+TablesyncWorkerMain(Datum main_arg)
+{
+	int			worker_slot = DatumGetInt32(main_arg);
+	char		originname[NAMEDATALEN];
+	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
+	char	   *myslotname = NULL;
+	WalRcvStreamOptions options;
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
 
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-
-		/* Is the use of a password mandatory? */
-		must_use_password = MySubscription->passwordrequired &&
-			!superuser_arg(MySubscription->owner);
-
-		/* Note that the superuser_arg call can access the DB */
-		CommitTransactionCommand();
+	elog(LOG, "logical replication table synchronization worker has started");
 
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												must_use_password,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
+	/* Attach to slot */
+	logicalrep_worker_attach(worker_slot);
 
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+	/* Setup signal handling */
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
 
-		set_apply_error_context_origin(originname);
-	}
+	/*
+	 * We don't currently need any ResourceOwner in a walreceiver process, but
+	 * if we did, we could call CreateAuxProcessResourceOwner here.
+	 */
+
+	/* Initialise stats to a sanish value */
+	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
+		MyLogicalRepWorker->reply_time = GetCurrentTimestamp();
+
+	/* Load the libpq-specific functions */
+	load_file("libpqwalreceiver", false);
+
+	InitializeLogRepWorker();
+
+	/* Connect to the origin and start the replication. */
+	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
+		 MySubscription->conninfo);
 
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * The loop where worker does its job. It loops until there is no relation
+	 * left to sync.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
+	for (;;)
 	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
+		run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
+		if (IsTransactionState())
+			CommitTransactionCommand();
+
+		if (MyLogicalRepWorker->ready_to_reuse)
+		{
+		/* This transaction will be committed by clean_sync_worker. */
+		StartTransactionCommand();
 
-	if (!am_tablesync_worker())
-	{
 		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
+		rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+
+		foreach(lc, rstates)
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
 
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * Pick the table for the next run if it is not already picked up
+			 * by another worker.
+			 *
+			 * Take exclusive lock to prevent any other sync worker from picking
+			 * the same table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
 		}
+
+		/*
+		 * If a relation with INIT state is assigned, clean up the worker for
+		 * the next iteration.
+		 *
+		 * If there is no more work left for this worker, break the loop to
+		 * exit.
+		 */
+		if ( MyLogicalRepWorker->relstate == SUBREL_STATE_INIT)
+			clean_sync_worker();
 		else
-		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-		}
+			break;
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-						MySubscription->name,
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-						MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-						"?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+		/* If not exited yet, then the worker will sync another table. */
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+						MySubscription->name, get_rel_name(MyLogicalRepWorker->relid), MyLogicalRepWorker->relid)));
+		CommitTransactionCommand();
+		}
 	}
 
-	/* Run the main loop. */
-	start_apply(origin_startpos);
-
-	proc_exit(0);
+	sync_worker_exit();
 }
 
 /*
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index 39588da79f..bbd71d0b42 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -18,6 +18,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending;
 
 extern void ApplyWorkerMain(Datum main_arg);
 extern void ParallelApplyWorkerMain(Datum main_arg);
+extern void TablesyncWorkerMain(Datum main_arg);
 
 extern bool IsLogicalWorker(void);
 extern bool IsLogicalParallelApplyWorker(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index b57eed052f..412195f143 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -76,6 +76,12 @@ typedef struct LogicalRepWorker
 	/* Indicates whether apply can be performed in parallel. */
 	bool		parallel_apply;
 
+	/*
+	 * Used to indicate whether sync worker is ready for being reused
+	 * to sync another relation.
+	 */
+	bool		ready_to_reuse;
+
 	/* Stats. */
 	XLogRecPtr	last_lsn;
 	TimestampTz last_send_time;
@@ -265,7 +271,7 @@ extern void maybe_reread_subscription(void);
 
 extern void stream_cleanup_files(Oid subid, TransactionId xid);
 
-extern void InitializeApplyWorker(void);
+extern void InitializeLogRepWorker(void);
 
 extern void store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn);
 
@@ -273,6 +279,8 @@ extern void store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn);
 extern void apply_error_callback(void *arg);
 extern void set_apply_error_context_origin(char *originname);
 
+extern void sync_worker_exit(void);
+
 /* Parallel apply worker setup and interactions */
 extern void pa_allocate_worker(TransactionId xid);
 extern ParallelApplyWorkerInfo *pa_find_worker(TransactionId xid);
-- 
2.25.1

From 9655559b3ebcc6fc87174764a648c858d2053f22 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 2/3] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy
a relation and then apply changes by consuming from replication slot.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 ++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 104 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 289 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index b11d9a6ba3..f7d5acee93 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2595,6 +2595,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index 052505e46f..33075016a8 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -82,6 +82,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -98,7 +100,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -982,6 +985,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 7e1f677f7a..4623717013 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -476,6 +476,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, must obtain a snapshot able to read all tables;
+ *  	if false, one that can read only catalogs is acceptable.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -494,6 +498,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -502,6 +507,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -571,8 +577,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 55a24c02c9..85c5cdb633 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 6035cf4816..c5b2d5b61f 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -486,6 +486,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 45b8b3684f..d335c88eda 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1284,7 +1285,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1336,6 +1337,98 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	CheckLogicalDecodingRequirements();
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consistent WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1864,6 +1957,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 281626fa6f..f01e5f23a7 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -386,6 +386,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -403,6 +413,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -437,6 +448,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index b4058b88c3..80ace35868 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2343,6 +2343,7 @@ ReplicationSlotInvalidationCause
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3818,6 +3819,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.25.1

From d5dde676d3ff4e1398609ef8937cf1604b70c5c6 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 3/3] Reuse Replication Slot and Origin in Tablesync

This commit allows reusing replication slots and origins during tablesync.

Earlier, a tablesync worker was creating a new replication slot and origin
each time it syncs a new table. With this patch, replication
slots/origins can be reusable for tablesync.

This reduces the overhead of creating/dropping replication slots and origins
and improves tablesync speed significantly especially for empty or small tables.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create those first. Otherwise the worker reuses
slot and origin created by the same worker in previous iterations earlier.
Tables in FINISHEDCOPY are expected to have a replication slot and origin.
Slot and origin names for such tables are persisted in
pg_subscription_rel catalog. Tablesync worker can fetch them and proceed
with existing slot and origin of FINISHEDCOPY tables and does not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 247 ++++++++++-
 src/backend/commands/subscriptioncmds.c       | 229 +++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   9 +-
 src/backend/replication/logical/tablesync.c   | 382 +++++++++++++-----
 src/backend/replication/logical/worker.c      | 142 ++++---
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  14 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  23 +-
 11 files changed, 852 insertions(+), 237 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 5240840552..0a33897a37 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8040,6 +8040,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8124,6 +8137,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index d07f88ce28..152fdaa310 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -108,6 +108,14 @@ GetSubscription(Oid subid, bool missing_ok)
 								   Anum_pg_subscription_suborigin);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -199,6 +207,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -228,7 +274,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -257,6 +303,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -269,6 +325,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -299,14 +409,56 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -318,6 +470,85 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 8b3de032ee..7941f6281e 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -708,6 +708,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -771,7 +772,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -862,6 +863,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
 	bool		must_use_password;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -941,7 +944,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -965,6 +968,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -991,13 +995,36 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
 				 * tablesync origin.
 				 */
-				if (state != SUBREL_STATE_READY)
+				if (state != SUBREL_STATE_READY &&
+					state != SUBREL_STATE_SYNCDONE)
 				{
 					char		originname[NAMEDATALEN];
 
@@ -1025,31 +1052,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1472,6 +1492,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1544,6 +1565,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1593,6 +1622,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1604,36 +1635,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1686,39 +1707,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1741,6 +1740,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2153,6 +2217,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2163,18 +2228,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 50aa1386d5..ffa27d7111 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										InvalidRepSlotId);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 3e89366e7c..e040249d93 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -437,6 +440,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->ready_to_reuse = false;
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
@@ -1161,7 +1165,8 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID,
+										 InvalidRepSlotId);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 0a813bc371..3d51e732bd 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -275,40 +275,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication tracking origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -317,32 +306,78 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
-		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/*
+			 * Drop replication origin
+			 *
+			 * There is a chance that the user is concurrently performing refresh
+			 * for the subscription where we remove the table state and its origin
+			 * or the apply worker would have removed this origin. So passing
+			 * missing_ok = true.
+			 */
+			replorigin_drop_by_name(originname, true, false);
+		}
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+		ereport(DEBUG2,
+			(errmsg("process_syncing_tables_for_sync: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					"NULL", "NULL", MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
+		StartTransactionCommand();
 
 		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+										   MyLogicalRepWorker->relid,
+										   originname,
+										   sizeof(originname));
 
 		/*
 		 * Sync worker is cleaned at this point. It's ready to sync next table,
@@ -352,8 +387,8 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		MyLogicalRepWorker->ready_to_reuse = true;
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 	}
-
-	SpinLockRelease(&MyLogicalRepWorker->relmutex);
+	else
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 }
 
 /*
@@ -437,6 +472,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -457,18 +493,31 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+				ereport(DEBUG2,
+					(errmsg("process_syncing_tables_for_apply: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+							"NULL", "NULL", rstate->state,
+							rstate->relid, MyLogicalRepWorker->subid)));
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -557,12 +606,25 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+						ereport(DEBUG2,
+								(errmsg("process_syncing_tables_for_apply: incremented lastusedid to %lld for subscription %u",
+										(long long) MySubscription->lastusedid, MySubscription->oid)));
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1185,8 +1247,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1197,11 +1259,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1225,6 +1287,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
 	bool		must_use_password;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1253,7 +1316,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1279,12 +1342,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1298,10 +1375,53 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+			ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL", "NULL", MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		memcpy(slotname, prev_slotname, NAMEDATALEN);
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1321,7 +1441,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1329,11 +1451,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+
+	/*
+	 * Refresh the originname in case of having non-existing origin
+	 * from previous failed sync attempts.
+	 * If that's the case, it should be removed from the catalog so far.
+	 * Then, we can continue by reusing the origin created by the current
+	 * worker instead of .
+	 */
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									MyLogicalRepWorker->relid,
+									originname,
+									sizeof(originname));
+
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					slotname, originname, MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
 	pgstat_report_stat(true);
 
 	StartTransactionCommand();
@@ -1371,6 +1513,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1386,48 +1529,95 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: created replication slot %s for subscription %u",
+						slotname, MyLogicalRepWorker->subid)));
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: reusing replication slot %s for relation %u in subscription %u",
+						slotname, MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1450,12 +1640,18 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					originname, slotname, SUBREL_STATE_FINISHEDCOPY,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 
 copy_table_done:
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index d5f87b84cb..69748d4656 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -464,8 +464,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -3760,6 +3768,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 		}
 	}
 
+	/* Pop the error context stack */
+	error_context_stack = errcallback.previous;
+	apply_error_context_stack = error_context_stack;
+
 	/*
 	 * If it's still not ready to reuse, this is probably an apply worker.
 	 * End streaming before exiting.
@@ -4521,6 +4533,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4637,11 +4652,13 @@ run_apply_worker(WalRcvStreamOptions *options,
 	replorigin_session_setup(originid, 0);
 	replorigin_session_origin = originid;
 	*origin_startpos = replorigin_session_get_progress(false);
-	CommitTransactionCommand();
 
 	/* Is the use of a password mandatory? */
-	must_use_password = MySubscription->passwordrequired &&
-		!superuser_arg(MySubscription->owner);
+		must_use_password = MySubscription->passwordrequired &&
+			!superuser_arg(MySubscription->owner);
+
+	CommitTransactionCommand();
+
 	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
 											must_use_password,
 											MySubscription->name, &err);
@@ -4901,60 +4918,87 @@ TablesyncWorkerMain(Datum main_arg)
 
 		if (MyLogicalRepWorker->ready_to_reuse)
 		{
-		/* This transaction will be committed by clean_sync_worker. */
-		StartTransactionCommand();
+			/* This transaction will be committed by clean_sync_worker. */
+			StartTransactionCommand();
 
-		/*
-		 * Check if any table whose relation state is still INIT. If a table
-		 * in INIT state is found, the worker will not be finished, it will be
-		 * reused instead.
-		 */
-		rstates = GetSubscriptionRelations(MySubscription->oid, true);
-		rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			/*
+			 * Check if any table whose relation state is still INIT. If a table
+			 * in INIT state is found, the worker will not be finished, it will be
+			 * reused instead.
+			 */
+			rstates = GetSubscriptionRelations(MySubscription->oid, true);
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
 
-		foreach(lc, rstates)
-		{
-			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+			foreach(lc, rstates)
+			{
+				memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+				/*
+				 * Pick the table for the next run if it is not already picked up
+				 * by another worker.
+				 *
+				 * Take exclusive lock to prevent any other sync worker from picking
+				 * the same table.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
+				if (rstate->state != SUBREL_STATE_SYNCDONE &&
+					!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+				{
+					/* Update worker state for the next table */
+					MyLogicalRepWorker->relid = rstate->relid;
+					MyLogicalRepWorker->relstate = rstate->state;
+					MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+					LWLockRelease(LogicalRepWorkerLock);
+					break;
+				}
+				LWLockRelease(LogicalRepWorkerLock);
+			}
 
 			/*
-			 * Pick the table for the next run if it is not already picked up
-			 * by another worker.
+			 * If a relation with INIT state is assigned, clean up the worker for
+			 * the next iteration.
 			 *
-			 * Take exclusive lock to prevent any other sync worker from picking
-			 * the same table.
+			 * If there is no more work left for this worker, drop repllication
+			 * slot and origin, then break the loop to exit.
 			 */
-			LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
-			if (rstate->state != SUBREL_STATE_SYNCDONE &&
-				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			if (MyLogicalRepWorker->relstate == SUBREL_STATE_INIT)
 			{
-				/* Update worker state for the next table */
-				MyLogicalRepWorker->relid = rstate->relid;
-				MyLogicalRepWorker->relstate = rstate->state;
-				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
-				LWLockRelease(LogicalRepWorkerLock);
-				break;
+				clean_sync_worker();
 			}
-			LWLockRelease(LogicalRepWorkerLock);
-		}
+			else
+			{
+				TimeLineID	tli;
 
-		/*
-		 * If a relation with INIT state is assigned, clean up the worker for
-		 * the next iteration.
-		 *
-		 * If there is no more work left for this worker, break the loop to
-		 * exit.
-		 */
-		if ( MyLogicalRepWorker->relstate == SUBREL_STATE_INIT)
-			clean_sync_worker();
-		else
-			break;
+				/*
+				 * It is important to give an error if we are unable to drop the
+				 * slot, otherwise, it won't be dropped till the corresponding
+				 * subscription is dropped. So passing missing_ok = false.
+				 */
+				if (MyLogicalRepWorker->created_slot)
+				{
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+					ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+				}
 
-		/* If not exited yet, then the worker will sync another table. */
-		StartTransactionCommand();
-		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
-						MySubscription->name, get_rel_name(MyLogicalRepWorker->relid), MyLogicalRepWorker->relid)));
-		CommitTransactionCommand();
+				/*
+				 * Drop replication origin before exiting.
+				 *
+				 * There is a chance that the user is concurrently performing refresh
+				 * for the subscription where we remove the table state and its origin
+				 * or the apply worker would have removed this origin. So passing
+				 * missing_ok = true.
+				 */
+				replorigin_drop_by_name(originname, true, false);
+
+				break;
+			}
+
+			/* If not exited yet, then the worker will sync another table. */
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid), MyLogicalRepWorker->relid)));
+			CommitTransactionCommand();
 		}
 	}
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 91d729d62d..493055ea98 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -108,6 +108,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -144,6 +147,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -164,6 +169,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..185164d75e 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,16 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index a8a89dc784..31b2c41893 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -237,8 +237,9 @@ extern bool InvalidateObsoleteReplicationSlots(ReplicationSlotInvalidationCause
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 412195f143..d6adb2fc39 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -239,7 +256,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(int slot_no, uint16 generation);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
@@ -334,4 +352,7 @@ am_parallel_apply_worker(void)
 	return isParallelApplyWorker(MyLogicalRepWorker);
 }
 
+/* Invalid identifier to be used for naming replication slots */
+#define InvalidRepSlotId	0
+
 #endif							/* WORKER_INTERNAL_H */
-- 
2.25.1

From 32fc88978366bf1dd3144ed714575daf8600ccaa Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Tue, 11 Apr 2023 14:24:22 +0300
Subject: [PATCH 1/3] Reuse Tablesync Workers

This commit allows reusing tablesync workers for syncing more than one
relation sequentially during their lifetime, instead of exiting after
only syncing one relation.

Before this commit, tablesync workers were capable of syncing only one
relation. For each table, a new sync worker was launched and the worker
would exit when the worker is done with the current table.

Now, tablesync workers are not only limited with one relation and can
move to another relation in the same subscription. This reduces the
overhead of launching a new background worker and exiting from that
worker for each relation.

A new tablesync worker gets launched only if the number of tablesync
workers for the subscription does not exceed
max_sync_workers_per_subscription. If there is a table needs to be synced,
a tablesync worker picks that up and syncs it.The worker continues to
picking new tables to sync until there is no table left for synchronization
in the subscription.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/postmaster/bgworker.c             |   3 +
 .../replication/logical/applyparallelworker.c |   2 +-
 src/backend/replication/logical/launcher.c    |   5 +-
 src/backend/replication/logical/tablesync.c   |  45 +-
 src/backend/replication/logical/worker.c      | 481 ++++++++++++------
 src/include/replication/logicalworker.h       |   1 +
 src/include/replication/worker_internal.h     |  11 +-
 7 files changed, 388 insertions(+), 160 deletions(-)

diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 0dd22b2351..5609919edf 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -131,6 +131,9 @@ static const struct
 	},
 	{
 		"ParallelApplyWorkerMain", ParallelApplyWorkerMain
+	},
+	{
+		"TablesyncWorkerMain", TablesyncWorkerMain
 	}
 };
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 82c1ddcdcb..f16e2377bf 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -942,7 +942,7 @@ ParallelApplyWorkerMain(Datum main_arg)
 	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
 		MyLogicalRepWorker->reply_time = 0;
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 87b5593d2d..86ee1f25c2 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -440,6 +440,7 @@ retry:
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
+	worker->ready_to_reuse = false;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
 	TIMESTAMP_NOBEGIN(worker->last_recv_time);
@@ -460,8 +461,10 @@ retry:
 
 	if (is_parallel_apply_worker)
 		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ParallelApplyWorkerMain");
-	else
+	else if (!OidIsValid(relid))
 		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
+	else
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "TablesyncWorkerMain");
 
 	if (OidIsValid(relid))
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index c56d42dcd2..524be7088a 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -128,11 +128,10 @@ static bool FetchTableStates(bool *started_tx);
 static StringInfo copybuf = NULL;
 
 /*
- * Exit routine for synchronization worker.
+ * Prepares the synchronization worker for reuse or exit.
  */
-static void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+void
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -144,18 +143,28 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/* And flush all writes. */
-	XLogFlush(GetXLogWriteRecPtr());
-
-	StartTransactionCommand();
-	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
-	CommitTransactionCommand();
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	walrcv_disconnect(LogRepWorkerWalRcvConn);
+	LogRepWorkerWalRcvConn = NULL;
 
 	/* Find the leader apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+void
+pg_attribute_noreturn()
+sync_worker_exit(void)
+{
+	clean_sync_worker();
+
+	/* And flush all writes. */
+	XLogFlush(GetXLogWriteRecPtr());
 
 	/* Stop gracefully */
 	proc_exit(0);
@@ -378,7 +387,13 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 */
 		replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+		/*
+		 * Sync worker is cleaned at this point. It's ready to sync next table,
+		 * if needed.
+		 */
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->ready_to_reuse = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
@@ -1275,7 +1290,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		case SUBREL_STATE_SYNCDONE:
 		case SUBREL_STATE_READY:
 		case SUBREL_STATE_UNKNOWN:
-			finish_sync_worker();	/* doesn't return */
+			sync_worker_exit();	/* doesn't return */
 	}
 
 	/* Calculate the name of the tablesync slot. */
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 78926f8647..8ca6078db4 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -392,6 +392,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -3617,6 +3618,27 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can go into process_syncing_tables_for_sync early.
+				 * Before we were able to reuse tablesync workers, that
+				 * process_syncing_tables_for_sync call would exit the worker
+				 * instead of preparing for reuse. Now that tablesync workers
+				 * can be reused and process_syncing_tables_for_sync is not
+				 * responsible for exiting. We need to take care of memory
+				 * contexts here before moving to sync the nex table or exit.
+				 */
+				if (MyLogicalRepWorker->ready_to_reuse)
+				{
+					MemoryContextResetAndDeleteChildren(ApplyMessageContext);
+					MemoryContextSwitchTo(TopMemoryContext);
+
+					/* Pop the error context stack */
+					error_context_stack = errcallback.previous;
+					apply_error_context_stack = error_context_stack;
+					return;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3636,6 +3658,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+			if (MyLogicalRepWorker->ready_to_reuse)
+			{
+				endofstream = true;
+			}
 		}
 
 		/* Cleanup the memory. */
@@ -3734,12 +3760,15 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 		}
 	}
 
-	/* Pop the error context stack */
-	error_context_stack = errcallback.previous;
-	apply_error_context_stack = error_context_stack;
-
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * If it's still not ready to reuse, this is probably an apply worker.
+	 * End streaming before exiting.
+	 */
+	if (!MyLogicalRepWorker->ready_to_reuse)
+	{
+		/* All done */
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
 }
 
 /*
@@ -4324,6 +4353,56 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* stream_build_options
+  * 	Build logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+stream_build_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4436,13 +4515,154 @@ start_apply(XLogRecPtr origin_startpos)
 }
 
 /*
- * Common initialization for leader apply worker and parallel apply worker.
+ * Runs the tablesync worker.
+ * It starts table sync. After successful sync,
+ * builds streaming options and starts streaming.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Set this to false for safety, in case we're already reusing the worker */
+	MyLogicalRepWorker->ready_to_reuse = false;
+
+	start_table_sync(origin_startpos, &slotname);
+
+	/*
+	 * Allocate the origin name in long-lived context for error context
+	 * message.
+	 */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+	CommitTransactionCommand();
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/* Start normal logical streaming replication. */
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	/* Start applying changes to catcup. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Runs the apply worker.
+ * It sets up replication origin, the streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	/* This is the leader apply worker */
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+	bool		must_use_password;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard
+	 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+	 * crash if slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+	CommitTransactionCommand();
+
+	/* Is the use of a password mandatory? */
+	must_use_password = MySubscription->passwordrequired &&
+		!superuser_arg(MySubscription->owner);
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											must_use_password,
+											MySubscription->name, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it
+	 * does some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	stream_build_options(options, slotname, origin_startpos);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		/* Start streaming with two_phase enabled */
+		options->proto.logical.twophase = true;
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+	else
+	{
+		walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+
+	/* Run the main loop. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Common initialization for logical replication workers; leader apply worker,
+ * parallel apply worker and tablesync worker.
  *
  * Initialize the database connection, in-memory subscription and necessary
  * config options.
  */
 void
-InitializeApplyWorker(void)
+InitializeLogRepWorker(void)
 {
 	MemoryContext oldctx;
 
@@ -4506,9 +4726,10 @@ InitializeApplyWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started",
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has started",
 						MySubscription->name,
-						get_rel_name(MyLogicalRepWorker->relid))));
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
 	else
 		ereport(LOG,
 		/* translator: first %s is the name of logical replication worker */
@@ -4527,7 +4748,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	InitializingApplyWorker = true;
 
@@ -4551,7 +4771,7 @@ ApplyWorkerMain(Datum main_arg)
 	/* Load the libpq-specific functions */
 	load_file("libpqwalreceiver", false);
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
@@ -4559,165 +4779,142 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
+	/*
+	 * Setup callback for syscache so that we know when something changes in
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
+	 */
+	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
+								  invalidate_syncing_table_states,
+								  (Datum) 0);
 
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-		bool		must_use_password;
+	/* This is leader apply worker */
+	run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 
-		myslotname = MySubscription->slotname;
+	proc_exit(0);
+}
 
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
+/* Logical Replication Tablesync worker entry point */
+void
+TablesyncWorkerMain(Datum main_arg)
+{
+	int			worker_slot = DatumGetInt32(main_arg);
+	char		originname[NAMEDATALEN];
+	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
+	char	   *myslotname = NULL;
+	WalRcvStreamOptions options;
+	List	   *rstates;
+	SubscriptionRelState *rstate;
+	ListCell   *lc;
 
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-
-		/* Is the use of a password mandatory? */
-		must_use_password = MySubscription->passwordrequired &&
-			!superuser_arg(MySubscription->owner);
-
-		/* Note that the superuser_arg call can access the DB */
-		CommitTransactionCommand();
+	elog(LOG, "logical replication table synchronization worker has started");
 
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												must_use_password,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
+	/* Attach to slot */
+	logicalrep_worker_attach(worker_slot);
 
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+	/* Setup signal handling */
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
 
-		set_apply_error_context_origin(originname);
-	}
+	/*
+	 * We don't currently need any ResourceOwner in a walreceiver process, but
+	 * if we did, we could call CreateAuxProcessResourceOwner here.
+	 */
+
+	/* Initialise stats to a sanish value */
+	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
+		MyLogicalRepWorker->reply_time = GetCurrentTimestamp();
+
+	/* Load the libpq-specific functions */
+	load_file("libpqwalreceiver", false);
+
+	InitializeLogRepWorker();
+
+	/* Connect to the origin and start the replication. */
+	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
+		 MySubscription->conninfo);
 
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
-
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
-
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
-
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * The loop where worker does its job. It loops until there is no relation
+	 * left to sync.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
+	for (;;)
 	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
+		run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
+		if (IsTransactionState())
+			CommitTransactionCommand();
+
+		if (MyLogicalRepWorker->ready_to_reuse)
+		{
+		/* This transaction will be committed by clean_sync_worker. */
+		StartTransactionCommand();
 
-	if (!am_tablesync_worker())
-	{
 		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
+		 * Check if any table whose relation state is still INIT. If a table
+		 * in INIT state is found, the worker will not be finished, it will be
+		 * reused instead.
 		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
+		rstates = GetSubscriptionRelations(MySubscription->oid, true);
+		rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+
+		foreach(lc, rstates)
 		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
 
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
+			/*
+			 * Pick the table for the next run if it is not already picked up
+			 * by another worker.
+			 *
+			 * Take exclusive lock to prevent any other sync worker from picking
+			 * the same table.
+			 */
+			LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
+			if (rstate->state != SUBREL_STATE_SYNCDONE &&
+				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			{
+				/* Update worker state for the next table */
+				MyLogicalRepWorker->relid = rstate->relid;
+				MyLogicalRepWorker->relstate = rstate->state;
+				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+				LWLockRelease(LogicalRepWorkerLock);
+				break;
+			}
+			LWLockRelease(LogicalRepWorkerLock);
 		}
+
+		/*
+		 * If a relation with INIT state is assigned, clean up the worker for
+		 * the next iteration.
+		 *
+		 * If there is no more work left for this worker, break the loop to
+		 * exit.
+		 */
+		if ( MyLogicalRepWorker->relstate == SUBREL_STATE_INIT)
+			clean_sync_worker();
 		else
-		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-		}
+			break;
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-								 MySubscription->name,
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-								 "?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+		/* If not exited yet, then the worker will sync another table. */
+		StartTransactionCommand();
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+						MySubscription->name, get_rel_name(MyLogicalRepWorker->relid), MyLogicalRepWorker->relid)));
+		CommitTransactionCommand();
+		}
 	}
 
-	/* Run the main loop. */
-	start_apply(origin_startpos);
-
-	proc_exit(0);
+	sync_worker_exit();
 }
 
 /*
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index 39588da79f..bbd71d0b42 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -18,6 +18,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending;
 
 extern void ApplyWorkerMain(Datum main_arg);
 extern void ParallelApplyWorkerMain(Datum main_arg);
+extern void TablesyncWorkerMain(Datum main_arg);
 
 extern bool IsLogicalWorker(void);
 extern bool IsLogicalParallelApplyWorker(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 343e781896..242737e42b 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -76,6 +76,12 @@ typedef struct LogicalRepWorker
 	/* Indicates whether apply can be performed in parallel. */
 	bool		parallel_apply;
 
+	/*
+	 * Used to indicate whether sync worker is ready for being reused
+	 * to sync another relation.
+	 */
+	bool		ready_to_reuse;
+
 	/* Stats. */
 	XLogRecPtr	last_lsn;
 	TimestampTz last_send_time;
@@ -265,7 +271,7 @@ extern void maybe_reread_subscription(void);
 
 extern void stream_cleanup_files(Oid subid, TransactionId xid);
 
-extern void InitializeApplyWorker(void);
+extern void InitializeLogRepWorker(void);
 
 extern void store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn);
 
@@ -273,6 +279,9 @@ extern void store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn);
 extern void apply_error_callback(void *arg);
 extern void set_apply_error_context_origin(char *originname);
 
+extern void sync_worker_exit(void);
+extern void clean_sync_worker(void);
+
 /* Parallel apply worker setup and interactions */
 extern void pa_allocate_worker(TransactionId xid);
 extern ParallelApplyWorkerInfo *pa_find_worker(TransactionId xid);
-- 
2.25.1

From 41b575dcf8b3e3272d7dfaf76f6c2b56e4b70019 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 2/3] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy
a relation and then apply changes by consuming from replication slot.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 ++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 104 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 289 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index b11d9a6ba3..f7d5acee93 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2595,6 +2595,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index dc9c5c82d9..cf88a3b7d4 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -82,6 +82,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -98,7 +100,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -982,6 +985,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 41243d0187..3c11f8e1dd 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -476,6 +476,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, must obtain a snapshot able to read all tables;
+ *  	if false, one that can read only catalogs is acceptable.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -494,6 +498,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -502,6 +507,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -571,8 +577,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 55a24c02c9..85c5cdb633 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 6035cf4816..c5b2d5b61f 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -486,6 +486,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index d3a136b6f5..87392dea1f 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1284,7 +1285,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1336,6 +1337,98 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	CheckLogicalDecodingRequirements();
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consistent WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1864,6 +1957,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 281626fa6f..f01e5f23a7 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -386,6 +386,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -403,6 +413,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -437,6 +448,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 260854747b..985a4ab7fd 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2348,6 +2348,7 @@ ReplicationSlotInvalidationCause
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3853,6 +3854,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.25.1

From f85b562995feaec25d0755d8b55d473c8cff8276 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 3/3] Reuse Replication Slot and Origin in Tablesync

This commit allows reusing replication slots and origins during tablesync.

Earlier, a tablesync worker was creating a new replication slot and origin
each time it syncs a new table. With this patch, replication
slots/origins can be reusable for tablesync.

This reduces the overhead of creating/dropping replication slots and origins
and improves tablesync speed significantly especially for empty or small tables.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create those first. Otherwise the worker reuses
slot and origin created by the same worker in previous iterations earlier.
Tables in FINISHEDCOPY are expected to have a replication slot and origin.
Slot and origin names for such tables are persisted in
pg_subscription_rel catalog. Tablesync worker can fetch them and proceed
with existing slot and origin of FINISHEDCOPY tables and does not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 247 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 229 +++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   9 +-
 src/backend/replication/logical/tablesync.c   | 378 +++++++++++++-----
 src/backend/replication/logical/worker.c      | 142 ++++---
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  14 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  23 +-
 11 files changed, 850 insertions(+), 235 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index ed32ca0349..0183ecd21e 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8030,6 +8030,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8114,6 +8127,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index d07f88ce28..152fdaa310 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -108,6 +108,14 @@ GetSubscription(Oid subid, bool missing_ok)
 								   Anum_pg_subscription_suborigin);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -199,6 +207,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -228,7 +274,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -257,6 +303,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -269,6 +325,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -299,14 +409,56 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -318,6 +470,85 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 1c88c2bccb..7cec23a806 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -710,6 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -773,7 +774,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -864,6 +865,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
 	bool		must_use_password;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -943,7 +946,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -967,6 +970,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -993,13 +997,36 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
 				 * tablesync origin.
 				 */
-				if (state != SUBREL_STATE_READY)
+				if (state != SUBREL_STATE_READY &&
+					state != SUBREL_STATE_SYNCDONE)
 				{
 					char		originname[NAMEDATALEN];
 
@@ -1027,31 +1054,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1474,6 +1494,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1546,6 +1567,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1595,6 +1624,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1606,36 +1637,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1688,39 +1709,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1743,6 +1742,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2155,6 +2219,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2165,18 +2230,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index f16e2377bf..a57f178087 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										InvalidRepSlotId);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 86ee1f25c2..9ab40fa4b8 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -437,6 +440,7 @@ retry:
 	worker->relid = relid;
 	worker->relstate = SUBREL_STATE_UNKNOWN;
 	worker->relstate_lsn = InvalidXLogRecPtr;
+	worker->ready_to_reuse = false;
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
@@ -1179,7 +1183,8 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID,
+										 InvalidRepSlotId);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 524be7088a..2d50b9a8a6 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -318,40 +318,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication tracking origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -360,32 +349,78 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
-		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/*
+			 * Drop replication origin
+			 *
+			 * There is a chance that the user is concurrently performing refresh
+			 * for the subscription where we remove the table state and its origin
+			 * or the apply worker would have removed this origin. So passing
+			 * missing_ok = true.
+			 */
+			replorigin_drop_by_name(originname, true, false);
+		}
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+		ereport(DEBUG2,
+			(errmsg("process_syncing_tables_for_sync: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					"NULL", "NULL", MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
+
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
+
+		StartTransactionCommand();
 
 		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+										   MyLogicalRepWorker->relid,
+										   originname,
+										   sizeof(originname));
 
 		/*
 		 * Sync worker is cleaned at this point. It's ready to sync next table,
@@ -480,6 +515,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -500,18 +536,31 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+				ereport(DEBUG2,
+					(errmsg("process_syncing_tables_for_apply: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+							"NULL", "NULL", rstate->state,
+							rstate->relid, MyLogicalRepWorker->subid)));
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -600,12 +649,25 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+						ereport(DEBUG2,
+								(errmsg("process_syncing_tables_for_apply: incremented lastusedid to %lld for subscription %u",
+										(long long) MySubscription->lastusedid, MySubscription->oid)));
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1228,8 +1290,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1240,11 +1302,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1268,6 +1330,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	char		originname[NAMEDATALEN];
 	RepOriginId originid;
 	bool		must_use_password;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1296,7 +1359,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1322,12 +1385,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1341,10 +1418,53 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+			ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL", "NULL", MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		memcpy(slotname, prev_slotname, NAMEDATALEN);
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1364,7 +1484,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1372,11 +1494,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+
+	/*
+	 * Refresh the originname in case of having non-existing origin
+	 * from previous failed sync attempts.
+	 * If that's the case, it should be removed from the catalog so far.
+	 * Then, we can continue by reusing the origin created by the current
+	 * worker instead of .
+	 */
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									MyLogicalRepWorker->relid,
+									originname,
+									sizeof(originname));
+
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					slotname, originname, MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
 	pgstat_report_stat(true);
 
 	StartTransactionCommand();
@@ -1414,6 +1556,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						GetUserNameFromId(GetUserId(), true),
 						RelationGetRelationName(rel))));
 
+
 	/*
 	 * Start a transaction in the remote node in REPEATABLE READ mode.  This
 	 * ensures that both the replication slot we create (see below) and the
@@ -1429,48 +1572,95 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: created replication slot %s for subscription %u",
+						slotname, MyLogicalRepWorker->subid)));
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: reusing replication slot %s for relation %u in subscription %u",
+						slotname, MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/* Now do the initial data copy */
 	PushActiveSnapshot(GetTransactionSnapshot());
 	copy_table(rel);
@@ -1493,12 +1683,18 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					originname, slotname, SUBREL_STATE_FINISHEDCOPY,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 
 copy_table_done:
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 8ca6078db4..f7199c434d 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -464,8 +464,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -3760,6 +3768,10 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 		}
 	}
 
+	/* Pop the error context stack */
+	error_context_stack = errcallback.previous;
+	apply_error_context_stack = error_context_stack;
+
 	/*
 	 * If it's still not ready to reuse, this is probably an apply worker.
 	 * End streaming before exiting.
@@ -4477,6 +4489,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4593,11 +4608,13 @@ run_apply_worker(WalRcvStreamOptions *options,
 	replorigin_session_setup(originid, 0);
 	replorigin_session_origin = originid;
 	*origin_startpos = replorigin_session_get_progress(false);
-	CommitTransactionCommand();
 
 	/* Is the use of a password mandatory? */
-	must_use_password = MySubscription->passwordrequired &&
-		!superuser_arg(MySubscription->owner);
+		must_use_password = MySubscription->passwordrequired &&
+			!superuser_arg(MySubscription->owner);
+
+	CommitTransactionCommand();
+
 	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
 											must_use_password,
 											MySubscription->name, &err);
@@ -4857,60 +4874,87 @@ TablesyncWorkerMain(Datum main_arg)
 
 		if (MyLogicalRepWorker->ready_to_reuse)
 		{
-		/* This transaction will be committed by clean_sync_worker. */
-		StartTransactionCommand();
+			/* This transaction will be committed by clean_sync_worker. */
+			StartTransactionCommand();
 
-		/*
-		 * Check if any table whose relation state is still INIT. If a table
-		 * in INIT state is found, the worker will not be finished, it will be
-		 * reused instead.
-		 */
-		rstates = GetSubscriptionRelations(MySubscription->oid, true);
-		rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+			/*
+			 * Check if any table whose relation state is still INIT. If a table
+			 * in INIT state is found, the worker will not be finished, it will be
+			 * reused instead.
+			 */
+			rstates = GetSubscriptionRelations(MySubscription->oid, true);
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
 
-		foreach(lc, rstates)
-		{
-			memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+			foreach(lc, rstates)
+			{
+				memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+				/*
+				 * Pick the table for the next run if it is not already picked up
+				 * by another worker.
+				 *
+				 * Take exclusive lock to prevent any other sync worker from picking
+				 * the same table.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
+				if (rstate->state != SUBREL_STATE_SYNCDONE &&
+					!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+				{
+					/* Update worker state for the next table */
+					MyLogicalRepWorker->relid = rstate->relid;
+					MyLogicalRepWorker->relstate = rstate->state;
+					MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+					LWLockRelease(LogicalRepWorkerLock);
+					break;
+				}
+				LWLockRelease(LogicalRepWorkerLock);
+			}
 
 			/*
-			 * Pick the table for the next run if it is not already picked up
-			 * by another worker.
+			 * If a relation with INIT state is assigned, clean up the worker for
+			 * the next iteration.
 			 *
-			 * Take exclusive lock to prevent any other sync worker from picking
-			 * the same table.
+			 * If there is no more work left for this worker, drop repllication
+			 * slot and origin, then break the loop to exit.
 			 */
-			LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
-			if (rstate->state != SUBREL_STATE_SYNCDONE &&
-				!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+			if (MyLogicalRepWorker->relstate == SUBREL_STATE_INIT)
 			{
-				/* Update worker state for the next table */
-				MyLogicalRepWorker->relid = rstate->relid;
-				MyLogicalRepWorker->relstate = rstate->state;
-				MyLogicalRepWorker->relstate_lsn = rstate->lsn;
-				LWLockRelease(LogicalRepWorkerLock);
-				break;
+				clean_sync_worker();
 			}
-			LWLockRelease(LogicalRepWorkerLock);
-		}
+			else
+			{
+				TimeLineID	tli;
 
-		/*
-		 * If a relation with INIT state is assigned, clean up the worker for
-		 * the next iteration.
-		 *
-		 * If there is no more work left for this worker, break the loop to
-		 * exit.
-		 */
-		if ( MyLogicalRepWorker->relstate == SUBREL_STATE_INIT)
-			clean_sync_worker();
-		else
-			break;
+				/*
+				 * It is important to give an error if we are unable to drop the
+				 * slot, otherwise, it won't be dropped till the corresponding
+				 * subscription is dropped. So passing missing_ok = false.
+				 */
+				if (MyLogicalRepWorker->created_slot)
+				{
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+					ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+				}
 
-		/* If not exited yet, then the worker will sync another table. */
-		StartTransactionCommand();
-		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
-						MySubscription->name, get_rel_name(MyLogicalRepWorker->relid), MyLogicalRepWorker->relid)));
-		CommitTransactionCommand();
+				/*
+				 * Drop replication origin before exiting.
+				 *
+				 * There is a chance that the user is concurrently performing refresh
+				 * for the subscription where we remove the table state and its origin
+				 * or the apply worker would have removed this origin. So passing
+				 * missing_ok = true.
+				 */
+				replorigin_drop_by_name(originname, true, false);
+
+				break;
+			}
+
+			/* If not exited yet, then the worker will sync another table. */
+			StartTransactionCommand();
+			ereport(LOG,
+					(errmsg("logical replication table synchronization worker for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+							MySubscription->name, get_rel_name(MyLogicalRepWorker->relid), MyLogicalRepWorker->relid)));
+			CommitTransactionCommand();
 		}
 	}
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 1d40eebc78..7e13f59847 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -108,6 +108,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -144,6 +147,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -164,6 +169,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..185164d75e 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,16 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index a8a89dc784..31b2c41893 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -237,8 +237,9 @@ extern bool InvalidateObsoleteReplicationSlots(ReplicationSlotInvalidationCause
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 242737e42b..6bbdf743e0 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -239,7 +256,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
@@ -335,4 +353,7 @@ am_parallel_apply_worker(void)
 	return isParallelApplyWorker(MyLogicalRepWorker);
 }
 
+/* Invalid identifier to be used for naming replication slots */
+#define InvalidRepSlotId	0
+
 #endif							/* WORKER_INTERNAL_H */
-- 
2.25.1

From cd58f9363144492ebc1112588b4ba6945c0f8ca0 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 4/4] Reuse Replication Slot and Origin in Tablesync

This commit allows reusing replication slots and origins during tablesync.

Earlier, a tablesync worker was creating a new replication slot and origin
each time it syncs a new table. With this patch, replication
slots/origins can be reusable for tablesync.

This reduces the overhead of creating/dropping replication slots and origins
and improves tablesync speed significantly especially for empty or small tables.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create those first. Otherwise the worker reuses
slot and origin created by the same worker in previous iterations earlier.
Tables in FINISHEDCOPY are expected to have a replication slot and origin.
Slot and origin names for such tables are persisted in
pg_subscription_rel catalog. Tablesync worker can fetch them and proceed
with existing slot and origin of FINISHEDCOPY tables and does not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 247 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 229 +++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   8 +-
 src/backend/replication/logical/tablesync.c   | 369 +++++++++++++-----
 src/backend/replication/logical/worker.c      |  62 ++-
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  14 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  23 +-
 11 files changed, 797 insertions(+), 198 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index ed32ca0349..0183ecd21e 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8030,6 +8030,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8114,6 +8127,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index d07f88ce28..152fdaa310 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -108,6 +108,14 @@ GetSubscription(Oid subid, bool missing_ok)
 								   Anum_pg_subscription_suborigin);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -199,6 +207,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -228,7 +274,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -257,6 +303,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -269,6 +325,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -299,14 +409,56 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -318,6 +470,85 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 54895ba929..9f5a295ddc 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -710,6 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -773,7 +774,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -864,6 +865,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
 	bool		must_use_password;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -943,7 +946,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -967,6 +970,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -993,13 +997,36 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
 				 * tablesync origin.
 				 */
-				if (state != SUBREL_STATE_READY)
+				if (state != SUBREL_STATE_READY &&
+					state != SUBREL_STATE_SYNCDONE)
 				{
 					char		originname[NAMEDATALEN];
 
@@ -1027,31 +1054,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1474,6 +1494,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1546,6 +1567,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1595,6 +1624,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1606,36 +1637,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1688,39 +1709,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1743,6 +1742,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2155,6 +2219,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2165,18 +2230,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index f16e2377bf..a57f178087 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										InvalidRepSlotId);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 0c6ce69c58..6dd5b9a7af 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -1180,7 +1183,8 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID,
+										 InvalidRepSlotId);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 37f073b968..7718287fe4 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -328,40 +328,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication tracking origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -370,32 +359,70 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
-		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/*
+			 * Drop replication origin
+			 *
+			 * There is a chance that the user is concurrently performing refresh
+			 * for the subscription where we remove the table state and its origin
+			 * or the apply worker would have removed this origin. So passing
+			 * missing_ok = true.
+			 */
+			replorigin_drop_by_name(originname, true, false);
+		}
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+		ereport(DEBUG2,
+			(errmsg("process_syncing_tables_for_sync: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					"NULL", "NULL", MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
 
 		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		StartTransactionCommand();
+		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+										   MyLogicalRepWorker->relid,
+										   originname,
+										   sizeof(originname));
 
 		/* Sync worker has completed synchronization of the current table. */
 		MyLogicalRepWorker->is_sync_completed = true;
@@ -492,6 +519,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -512,18 +540,31 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+				ereport(DEBUG2,
+					(errmsg("process_syncing_tables_for_apply: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+							"NULL", "NULL", rstate->state,
+							rstate->relid, MyLogicalRepWorker->subid)));
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -612,12 +653,25 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+						ereport(DEBUG2,
+								(errmsg("process_syncing_tables_for_apply: incremented lastusedid to %lld for subscription %u",
+										(long long) MySubscription->lastusedid, MySubscription->oid)));
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1240,8 +1294,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1252,11 +1306,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1282,6 +1336,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	UserContext ucxt;
 	bool		must_use_password;
 	bool		run_as_owner;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1316,7 +1371,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1338,12 +1393,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1357,10 +1426,53 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+			ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL", "NULL", MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		memcpy(slotname, prev_slotname, NAMEDATALEN);
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1380,7 +1492,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1388,11 +1502,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+
+	/*
+	 * Refresh the originname in case of having non-existing origin
+	 * from previous failed sync attempts.
+	 * If that's the case, it should be removed from the catalog so far.
+	 * Then, we can continue by reusing the origin created by the current
+	 * worker instead of .
+	 */
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									MyLogicalRepWorker->relid,
+									originname,
+									sizeof(originname));
+
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					slotname, originname, MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
 	pgstat_report_stat(true);
 
 	StartTransactionCommand();
@@ -1420,48 +1554,95 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: created replication slot %s for subscription %u",
+						slotname, MyLogicalRepWorker->subid)));
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: reusing replication slot %s for relation %u in subscription %u",
+						slotname, MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/*
 	 * Make sure that the copy command runs as the table owner, unless the
 	 * user has opted out of that behaviour.
@@ -1520,12 +1701,18 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					originname, slotname, SUBREL_STATE_FINISHEDCOPY,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 
 copy_table_done:
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 4a0f402ad4..0b89bc4c5d 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -464,8 +464,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -3768,7 +3776,8 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* Tablesync workers should end streaming before exiting the main loop
+	/*
+	 * Tablesync workers should end streaming before exiting the main loop
 	 * to drop replication slot. Only end streaming here for apply workers.
 	 */
 	if (!am_tablesync_worker())
@@ -4494,6 +4503,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4548,10 +4560,12 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname);
 
+	StartTransactionCommand();
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   originname_size);
+	CommitTransactionCommand();
 
 	set_apply_error_context_origin(originname);
 
@@ -4592,11 +4606,10 @@ run_apply_worker(WalRcvStreamOptions *options,
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 					errmsg("subscription has no replication slot set")));
 
-	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-									   originname, originname_size);
-
 	/* Setup replication origin tracking. */
 	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
 	originid = replorigin_by_name(originname, true);
 	if (!OidIsValid(originid))
 		originid = replorigin_create(originname);
@@ -4873,12 +4886,12 @@ TablesyncWorkerMain(Datum main_arg)
 				memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
 
 				/*
-				* Pick the table for the next run if it is not already picked up
-				* by another worker.
-				*
-				* Take exclusive lock to prevent any other sync worker from picking
-				* the same table.
-				*/
+				 * Pick the table for the next run if it is not already picked up
+				 * by another worker.
+				 *
+				 * Take exclusive lock to prevent any other sync worker from picking
+				 * the same table.
+				 */
 				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
 				if (rstate->state != SUBREL_STATE_SYNCDONE &&
 					!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
@@ -4908,7 +4921,32 @@ TablesyncWorkerMain(Datum main_arg)
 			}
 
 			if (!is_table_found)
+			{
+				TimeLineID	tli;
+
+				/*
+				 * It is important to give an error if we are unable to drop the
+				 * slot, otherwise, it won't be dropped till the corresponding
+				 * subscription is dropped. So passing missing_ok = false.
+				 */
+				if (MyLogicalRepWorker->created_slot)
+				{
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+					ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+				}
+
+				/*
+				 * Drop replication origin before exiting.
+				 *
+				 * There is a chance that the user is concurrently performing refresh
+				 * for the subscription where we remove the table state and its origin
+				 * or the apply worker would have removed this origin. So passing
+				 * missing_ok = true.
+				 */
+				replorigin_drop_by_name(originname, true, false);
+
 				break;
+			}
 		}
 	}
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 1d40eebc78..7e13f59847 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -108,6 +108,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -144,6 +147,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -164,6 +169,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..185164d75e 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,16 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index a8a89dc784..31b2c41893 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -237,8 +237,9 @@ extern bool InvalidateObsoleteReplicationSlots(ReplicationSlotInvalidationCause
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 1e9f8e6e72..24006240c9 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -239,7 +256,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
@@ -335,4 +353,7 @@ am_parallel_apply_worker(void)
 	return isParallelApplyWorker(MyLogicalRepWorker);
 }
 
+/* Invalid identifier to be used for naming replication slots */
+#define InvalidRepSlotId	0
+
 #endif							/* WORKER_INTERNAL_H */
-- 
2.25.1

From 81c38df18c7038dcfcabcc396e484d16b4680d51 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Mon, 5 Jun 2023 15:04:41 +0300
Subject: [PATCH 1/4] Refactor to split Apply and Tablesync Workers

Both apply and tablesync workers were using ApplyWorkerMain() as entry
point. As the name implies, ApplyWorkerMain() should be considered as
the main function for apply workers. Tablesync worker's path was hidden
and does not have enough in common to share the same main function with
apply worker.

Also; most of the code shared by both worker types are already combined
in LogicalRepApplyLoop(). There is no need to combine the rest in
ApplyWorkerMain() anymore.

This commit introduces TablesyncWorkerMain() as a new entry point for
tablesync workers and separates both type of workers from each other.
This aims to increase code readability and help to maintain logical
replication workers separately.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/postmaster/bgworker.c             |   3 +
 .../replication/logical/applyparallelworker.c |   2 +-
 src/backend/replication/logical/launcher.c    |  25 +-
 src/backend/replication/logical/tablesync.c   |   2 +-
 src/backend/replication/logical/worker.c      | 381 +++++++++++-------
 src/include/replication/logicalworker.h       |   1 +
 src/include/replication/worker_internal.h     |   4 +-
 7 files changed, 258 insertions(+), 160 deletions(-)

diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 0dd22b2351..5609919edf 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -131,6 +131,9 @@ static const struct
 	},
 	{
 		"ParallelApplyWorkerMain", ParallelApplyWorkerMain
+	},
+	{
+		"TablesyncWorkerMain", TablesyncWorkerMain
 	}
 };
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 82c1ddcdcb..f16e2377bf 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -942,7 +942,7 @@ ParallelApplyWorkerMain(Datum main_arg)
 	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
 		MyLogicalRepWorker->reply_time = 0;
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 87b5593d2d..c2bba3ba69 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -459,24 +459,27 @@ retry:
 	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
 
 	if (is_parallel_apply_worker)
+	{
 		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ParallelApplyWorkerMain");
-	else
-		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
-
-	if (OidIsValid(relid))
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
-				 "logical replication worker for subscription %u sync %u", subid, relid);
-	else if (is_parallel_apply_worker)
+				 "logical replication parallel apply worker for subscription %u", subid);
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
 				 "logical replication parallel apply worker for subscription %u", subid);
+	}
+	else if (OidIsValid(relid))
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "TablesyncWorkerMain");
+		snprintf(bgw.bgw_name, BGW_MAXLEN,
+				 "logical replication tablesync worker for subscription %u sync %u", subid, relid);
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication tablesync worker");
+	}
 	else
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
 				 "logical replication apply worker for subscription %u", subid);
-
-	if (is_parallel_apply_worker)
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication parallel worker");
-	else
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker");
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication apply worker");
+	}
 
 	bgw.bgw_restart_time = BGW_NEVER_RESTART;
 	bgw.bgw_notify_pid = MyProcPid;
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 6d461654ab..8125bbd170 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -131,7 +131,7 @@ static StringInfo copybuf = NULL;
 /*
  * Exit routine for synchronization worker.
  */
-static void
+void
 pg_attribute_noreturn()
 finish_sync_worker(void)
 {
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 0ee764d68f..b979a755ae 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -392,6 +392,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void set_stream_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -4330,6 +4331,69 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* set_stream_options
+  * 	Set logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+set_stream_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+		options->proto.logical.twophase = true;
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4442,13 +4506,134 @@ start_apply(XLogRecPtr origin_startpos)
 }
 
 /*
- * Common initialization for leader apply worker and parallel apply worker.
+ * Runs the tablesync worker.
+ * It starts syncing tables. After a successful sync,
+ * sets streaming options and starts streaming to catchup.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Start table synchronization. */
+	start_table_sync(origin_startpos, &slotname);
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	/* Start applying changes to catchup. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Runs the leader apply worker.
+ * It sets up replication origin, streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+	bool		must_use_password;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard
+	 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+	 * crash if slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+
+	/* Is the use of a password mandatory? */
+	must_use_password = MySubscription->passwordrequired &&
+		!superuser_arg(MySubscription->owner);
+
+	/* Note that the superuser_arg call can access the DB */
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											must_use_password,
+											MySubscription->name, &err);
+
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it
+	 * does some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+
+	/* Run the main loop. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Common initialization for logical replication workers; leader apply worker,
+ * parallel apply worker and tablesync worker.
  *
  * Initialize the database connection, in-memory subscription and necessary
  * config options.
  */
 void
-InitializeApplyWorker(void)
+InitializeLogRepWorker(void)
 {
 	MemoryContext oldctx;
 
@@ -4512,7 +4697,8 @@ InitializeApplyWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+						get_worker_name(),
 						MySubscription->name,
 						get_rel_name(MyLogicalRepWorker->relid))));
 	else
@@ -4533,7 +4719,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	InitializingApplyWorker = true;
 
@@ -4557,7 +4742,7 @@ ApplyWorkerMain(Datum main_arg)
 	/* Load the libpq-specific functions */
 	load_file("libpqwalreceiver", false);
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
@@ -4565,165 +4750,69 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-		bool		must_use_password;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-
-		/* Is the use of a password mandatory? */
-		must_use_password = MySubscription->passwordrequired &&
-			!superuser_arg(MySubscription->owner);
-
-		/* Note that the superuser_arg call can access the DB */
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												must_use_password,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
+	/* This is leader apply worker */
+	run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
+	proc_exit(0);
+}
 
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
+/* Logical Replication Tablesync worker entry point */
+void
+TablesyncWorkerMain(Datum main_arg)
+{
+	int			worker_slot = DatumGetInt32(main_arg);
+	char		originname[NAMEDATALEN];
+	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
+	char	   *myslotname = NULL;
+	WalRcvStreamOptions options;
+
+	/* Attach to slot */
+	logicalrep_worker_attach(worker_slot);
+
+	/* Setup signal handling */
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
 
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * We don't currently need any ResourceOwner in a walreceiver process, but
+	 * if we did, we could call CreateAuxProcessResourceOwner here.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
 
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
+	/* Initialise stats to a sanish value */
+	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
+		MyLogicalRepWorker->reply_time = GetCurrentTimestamp();
 
-	if (!am_tablesync_worker())
-	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
-		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+	/* Load the libpq-specific functions */
+	load_file("libpqwalreceiver", false);
 
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
-		}
-		else
-		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-		}
+	InitializeLogRepWorker();
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-								 MySubscription->name,
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-								 "?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
+	/* Connect to the origin and start the replication. */
+	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
+		 MySubscription->conninfo);
 
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+	/*
+	 * Setup callback for syscache so that we know when something changes in
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
+	 */
+	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
+								  invalidate_syncing_table_states,
+								  (Datum) 0);
 
-	proc_exit(0);
+	run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+
+	finish_sync_worker();
 }
 
 /*
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index 39588da79f..bbd71d0b42 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -18,6 +18,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending;
 
 extern void ApplyWorkerMain(Datum main_arg);
 extern void ParallelApplyWorkerMain(Datum main_arg);
+extern void TablesyncWorkerMain(Datum main_arg);
 
 extern bool IsLogicalWorker(void);
 extern bool IsLogicalParallelApplyWorker(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 343e781896..7aba034774 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -265,7 +265,7 @@ extern void maybe_reread_subscription(void);
 
 extern void stream_cleanup_files(Oid subid, TransactionId xid);
 
-extern void InitializeApplyWorker(void);
+extern void InitializeLogRepWorker(void);
 
 extern void store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn);
 
@@ -307,6 +307,8 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
+extern void finish_sync_worker(void);
+
 static inline bool
 am_tablesync_worker(void)
 {
-- 
2.25.1

From dcfc3094dbb40550a5055496d156a87b8e3e2065 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Mon, 5 Jun 2023 15:45:29 +0300
Subject: [PATCH 2/4] Reuse Tablesync Workers

This commit allows reusing tablesync workers for syncing more than one
table sequentially during their lifetime, instead of exiting after
only syncing one table.

Before this commit, tablesync workers were capable of syncing only one
table. For each table, a new sync worker was launched and that worker would
exit when done processing the table.

Now, tablesync workers are not limited to processing only one
table. When done, they can move to processing another table in
the same subscription.

If there is a table that needs to be synced, an available tablesync
worker picks up that table and syncs it. Each tablesync worker
continues to pick new tables to sync until there are no tables left
requiring synchronization. If there was no available worker to
process the table, then a new tablesync worker will be launched,
provided the number of tablesync workers for the subscription does not
exceed max_sync_workers_per_subscription.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/replication/logical/launcher.c  |   1 +
 src/backend/replication/logical/tablesync.c |  46 ++++++--
 src/backend/replication/logical/worker.c    | 110 +++++++++++++++++++-
 src/include/replication/worker_internal.h   |   7 ++
 4 files changed, 149 insertions(+), 15 deletions(-)

diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index c2bba3ba69..0c6ce69c58 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -440,6 +440,7 @@ retry:
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
+	worker->is_sync_completed = false;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
 	TIMESTAMP_NOBEGIN(worker->last_recv_time);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8125bbd170..37f073b968 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -129,11 +129,10 @@ static bool FetchTableStates(bool *started_tx);
 static StringInfo copybuf = NULL;
 
 /*
- * Exit routine for synchronization worker.
+ * Prepares the synchronization worker for reuse or exit.
  */
 void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -145,19 +144,38 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	if (LogRepWorkerWalRcvConn != NULL)
+	{
+		walrcv_disconnect(LogRepWorkerWalRcvConn);
+		LogRepWorkerWalRcvConn = NULL;
+	}
+
+	/* Find the leader apply worker and signal it. */
+	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
 	/* And flush all writes. */
 	XLogFlush(GetXLogWriteRecPtr());
 
 	StartTransactionCommand();
 	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
+			(errmsg("logical replication table synchronization worker for subscription \"%s\" has finished",
+					MySubscription->name)));
 	CommitTransactionCommand();
 
-	/* Find the leader apply worker and signal it. */
-	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
-
 	/* Stop gracefully */
 	proc_exit(0);
 }
@@ -379,7 +397,15 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 */
 		replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+		/* Sync worker has completed synchronization of the current table. */
+		MyLogicalRepWorker->is_sync_completed = true;
+
+		ereport(LOG,
+		(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has finished",
+				MySubscription->name,
+				get_rel_name(MyLogicalRepWorker->relid),
+				MyLogicalRepWorker->relid)));
+		CommitTransactionCommand();
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index b979a755ae..4a0f402ad4 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3624,6 +3624,20 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can call process_syncing_tables_for_sync.
+				 *
+				 * process_syncing_tables_for_sync decides whether the sync of the
+				 * current table is completed. If it is completed, streaming must
+				 * be already ended. So, we can break the loop.
+				 */
+				if (MyLogicalRepWorker->is_sync_completed)
+				{
+					endofstream = true;
+					break;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3643,6 +3657,15 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+
+			/*
+			 * If is_sync_completed is true, this means that the tablesync worker
+			 * is done with synchronization. Streaming has already been ended by
+			 * process_syncing_tables_for_sync. We should move to the next table
+			 * if needed, or exit.
+			 */
+			if (MyLogicalRepWorker->is_sync_completed)
+				endofstream = true;
 		}
 
 		/* Cleanup the memory. */
@@ -3745,8 +3768,11 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/* Tablesync workers should end streaming before exiting the main loop
+	 * to drop replication slot. Only end streaming here for apply workers.
+	 */
+	if (!am_tablesync_worker())
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
 }
 
 /*
@@ -4517,6 +4543,8 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 					 int originname_size,
 					 XLogRecPtr *origin_startpos)
 {
+	MyLogicalRepWorker->is_sync_completed = false;
+
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname);
 
@@ -4697,10 +4725,11 @@ InitializeLogRepWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" with relid %u has started",
 						get_worker_name(),
 						MySubscription->name,
-						get_rel_name(MyLogicalRepWorker->relid))));
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
 	else
 		ereport(LOG,
 		/* translator: first %s is the name of logical replication worker */
@@ -4810,7 +4839,78 @@ TablesyncWorkerMain(Datum main_arg)
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+	/*
+	 * The loop where worker does its job. It loops until there is no relation
+	 * left to sync.
+	 */
+	for (;;)
+	{
+		List	   *rstates;
+		SubscriptionRelState *rstate;
+		ListCell   *lc;
+		bool 	is_table_found = false;
+
+		run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+
+		if (IsTransactionState())
+			CommitTransactionCommand();
+
+		if (MyLogicalRepWorker->is_sync_completed)
+		{
+			/* This transaction will be committed by clean_sync_worker. */
+			StartTransactionCommand();
+
+			/*
+			 * Check if there is any table whose relation state is still INIT.
+			 * If a table in INIT state is found, the worker will not be finished,
+			 * it will be reused instead.
+			 */
+			rstates = GetSubscriptionRelations(MySubscription->oid, true);
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+
+			foreach(lc, rstates)
+			{
+				memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+				/*
+				* Pick the table for the next run if it is not already picked up
+				* by another worker.
+				*
+				* Take exclusive lock to prevent any other sync worker from picking
+				* the same table.
+				*/
+				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
+				if (rstate->state != SUBREL_STATE_SYNCDONE &&
+					!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+				{
+					/* Update worker state for the next table */
+					MyLogicalRepWorker->relid = rstate->relid;
+					MyLogicalRepWorker->relstate = rstate->state;
+					MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+					LWLockRelease(LogicalRepWorkerLock);
+
+					/* found a table for next iteration */
+					is_table_found = true;
+					clean_sync_worker();
+
+					StartTransactionCommand();
+					ereport(LOG,
+							(errmsg("%s for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+									get_worker_name(),
+									MySubscription->name,
+									get_rel_name(MyLogicalRepWorker->relid),
+									MyLogicalRepWorker->relid)));
+					CommitTransactionCommand();
+
+					break;
+				}
+				LWLockRelease(LogicalRepWorkerLock);
+			}
+
+			if (!is_table_found)
+				break;
+		}
+	}
 
 	finish_sync_worker();
 }
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 7aba034774..1e9f8e6e72 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -56,6 +56,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Indicates whether tablesync worker has completed sycning its assigned
+	 * table. If true, no need to continue with that table.
+	 */
+	bool		is_sync_completed;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -308,6 +314,7 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
 extern void finish_sync_worker(void);
+extern void clean_sync_worker(void);
 
 static inline bool
 am_tablesync_worker(void)
-- 
2.25.1

From 6a0fd4f6678fc6f5ee7f3792edf6ffcfcf10d26a Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 3/4] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy
a relation and then apply changes by consuming from replication slot.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 ++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 104 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 289 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index b11d9a6ba3..f7d5acee93 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2595,6 +2595,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index dc9c5c82d9..cf88a3b7d4 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -82,6 +82,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -98,7 +100,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -982,6 +985,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 41243d0187..3c11f8e1dd 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -476,6 +476,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, must obtain a snapshot able to read all tables;
+ *  	if false, one that can read only catalogs is acceptable.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -494,6 +498,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -502,6 +507,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -571,8 +577,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 55a24c02c9..85c5cdb633 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 6035cf4816..c5b2d5b61f 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -486,6 +486,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index d3a136b6f5..87392dea1f 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1284,7 +1285,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1336,6 +1337,98 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	CheckLogicalDecodingRequirements();
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consistent WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1864,6 +1957,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 281626fa6f..f01e5f23a7 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -386,6 +386,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -403,6 +413,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -437,6 +448,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 260854747b..985a4ab7fd 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2348,6 +2348,7 @@ ReplicationSlotInvalidationCause
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3853,6 +3854,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.25.1

From 490498f5abef6a898576adf4a0dec0deef80ce61 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 4/5] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy
a relation and then apply changes by consuming from replication slot.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 ++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 104 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 289 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index b11d9a6ba3..f7d5acee93 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2595,6 +2595,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index dc9c5c82d9..cf88a3b7d4 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -82,6 +82,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -98,7 +100,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -982,6 +985,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 41243d0187..3c11f8e1dd 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -476,6 +476,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, must obtain a snapshot able to read all tables;
+ *  	if false, one that can read only catalogs is acceptable.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -494,6 +498,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -502,6 +507,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -571,8 +577,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 55a24c02c9..85c5cdb633 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 6035cf4816..c5b2d5b61f 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -486,6 +486,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 429d00f2f0..974ab5ca35 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1284,7 +1285,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1336,6 +1337,98 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	CheckLogicalDecodingRequirements();
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consistent WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1870,6 +1963,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 281626fa6f..f01e5f23a7 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -386,6 +386,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -403,6 +413,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -437,6 +448,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 260854747b..985a4ab7fd 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2348,6 +2348,7 @@ ReplicationSlotInvalidationCause
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3853,6 +3854,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.27.0

From fd7065669432bf177aa6bdbef36d6dcbcf6dea75 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 5/5] Reuse Replication Slot and Origin in Tablesync

This commit allows reusing replication slots and origins during tablesync.

Earlier, a tablesync worker was creating a new replication slot and origin
each time it syncs a new table. With this patch, replication
slots/origins can be reusable for tablesync.

This reduces the overhead of creating/dropping replication slots and origins
and improves tablesync speed significantly especially for empty or small tables.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create those first. Otherwise the worker reuses
slot and origin created by the same worker in previous iterations earlier.
Tables in FINISHEDCOPY are expected to have a replication slot and origin.
Slot and origin names for such tables are persisted in
pg_subscription_rel catalog. Tablesync worker can fetch them and proceed
with existing slot and origin of FINISHEDCOPY tables and does not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 247 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 229 +++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   8 +-
 src/backend/replication/logical/tablesync.c   | 369 +++++++++++++-----
 src/backend/replication/logical/worker.c      |  62 ++-
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  14 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  23 +-
 11 files changed, 797 insertions(+), 198 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index ed32ca0349..0183ecd21e 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8030,6 +8030,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8114,6 +8127,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index d07f88ce28..152fdaa310 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -108,6 +108,14 @@ GetSubscription(Oid subid, bool missing_ok)
 								   Anum_pg_subscription_suborigin);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -199,6 +207,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -228,7 +274,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -257,6 +303,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -269,6 +325,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -299,14 +409,56 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -318,6 +470,85 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 54895ba929..9f5a295ddc 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -710,6 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -773,7 +774,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -864,6 +865,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
 	bool		must_use_password;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -943,7 +946,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -967,6 +970,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -993,13 +997,36 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
 				 * tablesync origin.
 				 */
-				if (state != SUBREL_STATE_READY)
+				if (state != SUBREL_STATE_READY &&
+					state != SUBREL_STATE_SYNCDONE)
 				{
 					char		originname[NAMEDATALEN];
 
@@ -1027,31 +1054,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1474,6 +1494,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1546,6 +1567,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1595,6 +1624,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1606,36 +1637,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1688,39 +1709,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1743,6 +1742,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2155,6 +2219,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2165,18 +2230,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index f16e2377bf..a57f178087 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										InvalidRepSlotId);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index c24e56a11e..d0d8d95a68 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -1180,7 +1183,8 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID,
+										 InvalidRepSlotId);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 30fe9b78ac..c032214959 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -328,40 +328,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication tracking origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -370,32 +359,70 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
-		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/*
+			 * Drop replication origin
+			 *
+			 * There is a chance that the user is concurrently performing refresh
+			 * for the subscription where we remove the table state and its origin
+			 * or the apply worker would have removed this origin. So passing
+			 * missing_ok = true.
+			 */
+			replorigin_drop_by_name(originname, true, false);
+		}
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+		ereport(DEBUG2,
+			(errmsg("process_syncing_tables_for_sync: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					"NULL", "NULL", MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
 
 		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		StartTransactionCommand();
+		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+										   MyLogicalRepWorker->relid,
+										   originname,
+										   sizeof(originname));
 
 		/* Sync worker has completed synchronization of the current table. */
 		MyLogicalRepWorker->is_sync_completed = true;
@@ -492,6 +519,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -512,18 +540,31 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+				ereport(DEBUG2,
+					(errmsg("process_syncing_tables_for_apply: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+							"NULL", "NULL", rstate->state,
+							rstate->relid, MyLogicalRepWorker->subid)));
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -612,12 +653,25 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+						ereport(DEBUG2,
+								(errmsg("process_syncing_tables_for_apply: incremented lastusedid to %lld for subscription %u",
+										(long long) MySubscription->lastusedid, MySubscription->oid)));
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1240,8 +1294,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1252,11 +1306,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1282,6 +1336,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	UserContext ucxt;
 	bool		must_use_password;
 	bool		run_as_owner;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1316,7 +1371,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1345,12 +1400,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1364,10 +1433,53 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+			ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL", "NULL", MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		memcpy(slotname, prev_slotname, NAMEDATALEN);
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1387,7 +1499,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1395,11 +1509,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+
+	/*
+	 * Refresh the originname in case of having non-existing origin
+	 * from previous failed sync attempts.
+	 * If that's the case, it should be removed from the catalog so far.
+	 * Then, we can continue by reusing the origin created by the current
+	 * worker instead of .
+	 */
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									MyLogicalRepWorker->relid,
+									originname,
+									sizeof(originname));
+
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					slotname, originname, MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
 	pgstat_report_stat(true);
 
 	StartTransactionCommand();
@@ -1427,48 +1561,95 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: created replication slot %s for subscription %u",
+						slotname, MyLogicalRepWorker->subid)));
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: reusing replication slot %s for relation %u in subscription %u",
+						slotname, MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/*
 	 * Make sure that the copy command runs as the table owner, unless the
 	 * user has opted out of that behaviour.
@@ -1527,12 +1708,18 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					originname, slotname, SUBREL_STATE_FINISHEDCOPY,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 
 copy_table_done:
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 8df960e343..c8f8b72aa7 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -464,8 +464,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -3770,7 +3778,8 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* Tablesync workers should end streaming before exiting the main loop
+	/*
+	 * Tablesync workers should end streaming before exiting the main loop
 	 * to drop replication slot. Only end streaming here for apply workers.
 	 */
 	if (!am_tablesync_worker())
@@ -4496,6 +4505,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4550,10 +4562,12 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname);
 
+	StartTransactionCommand();
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   originname_size);
+	CommitTransactionCommand();
 
 	set_apply_error_context_origin(originname);
 
@@ -4594,11 +4608,10 @@ run_apply_worker(WalRcvStreamOptions *options,
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 					errmsg("subscription has no replication slot set")));
 
-	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-									   originname, originname_size);
-
 	/* Setup replication origin tracking. */
 	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
 	originid = replorigin_by_name(originname, true);
 	if (!OidIsValid(originid))
 		originid = replorigin_create(originname);
@@ -4875,12 +4888,12 @@ TablesyncWorkerMain(Datum main_arg)
 				memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
 
 				/*
-				* Pick the table for the next run if it is not already picked up
-				* by another worker.
-				*
-				* Take exclusive lock to prevent any other sync worker from picking
-				* the same table.
-				*/
+				 * Pick the table for the next run if it is not already picked up
+				 * by another worker.
+				 *
+				 * Take exclusive lock to prevent any other sync worker from picking
+				 * the same table.
+				 */
 				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
 				if (rstate->state != SUBREL_STATE_SYNCDONE &&
 					!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
@@ -4912,7 +4925,32 @@ TablesyncWorkerMain(Datum main_arg)
 			}
 
 			if (!is_table_found)
+			{
+				TimeLineID	tli;
+
+				/*
+				 * It is important to give an error if we are unable to drop the
+				 * slot, otherwise, it won't be dropped till the corresponding
+				 * subscription is dropped. So passing missing_ok = false.
+				 */
+				if (MyLogicalRepWorker->created_slot)
+				{
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+					ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+				}
+
+				/*
+				 * Drop replication origin before exiting.
+				 *
+				 * There is a chance that the user is concurrently performing refresh
+				 * for the subscription where we remove the table state and its origin
+				 * or the apply worker would have removed this origin. So passing
+				 * missing_ok = true.
+				 */
+				replorigin_drop_by_name(originname, true, false);
+
 				break;
+			}
 		}
 	}
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 1d40eebc78..7e13f59847 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -108,6 +108,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -144,6 +147,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -164,6 +169,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..185164d75e 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,16 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index a8a89dc784..31b2c41893 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -237,8 +237,9 @@ extern bool InvalidateObsoleteReplicationSlots(ReplicationSlotInvalidationCause
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 1e9f8e6e72..24006240c9 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -239,7 +256,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
@@ -335,4 +353,7 @@ am_parallel_apply_worker(void)
 	return isParallelApplyWorker(MyLogicalRepWorker);
 }
 
+/* Invalid identifier to be used for naming replication slots */
+#define InvalidRepSlotId	0
+
 #endif							/* WORKER_INTERNAL_H */
-- 
2.27.0

From 9ba4e50c8e9884e1da18216092cc2418c5cae3ae Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Mon, 5 Jun 2023 15:04:41 +0300
Subject: [PATCH 1/5] Refactor to split Apply and Tablesync Workers

Both apply and tablesync workers were using ApplyWorkerMain() as entry
point. As the name implies, ApplyWorkerMain() should be considered as
the main function for apply workers. Tablesync worker's path was hidden
and does not have enough in common to share the same main function with
apply worker.

Also; most of the code shared by both worker types are already combined
in LogicalRepApplyLoop(). There is no need to combine the rest in
ApplyWorkerMain() anymore.

This commit introduces TablesyncWorkerMain() as a new entry point for
tablesync workers and separates both type of workers from each other.
This aims to increase code readability and help to maintain logical
replication workers separately.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/postmaster/bgworker.c             |   3 +
 .../replication/logical/applyparallelworker.c |   2 +-
 src/backend/replication/logical/launcher.c    |  25 +-
 src/backend/replication/logical/tablesync.c   |   2 +-
 src/backend/replication/logical/worker.c      | 381 +++++++++++-------
 src/include/replication/logicalworker.h       |   1 +
 src/include/replication/worker_internal.h     |   4 +-
 7 files changed, 258 insertions(+), 160 deletions(-)

diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 0dd22b2351..5609919edf 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -131,6 +131,9 @@ static const struct
 	},
 	{
 		"ParallelApplyWorkerMain", ParallelApplyWorkerMain
+	},
+	{
+		"TablesyncWorkerMain", TablesyncWorkerMain
 	}
 };
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 82c1ddcdcb..f16e2377bf 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -942,7 +942,7 @@ ParallelApplyWorkerMain(Datum main_arg)
 	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
 		MyLogicalRepWorker->reply_time = 0;
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 8395ae7b23..f562b1db41 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -459,24 +459,27 @@ retry:
 	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
 
 	if (is_parallel_apply_worker)
+	{
 		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ParallelApplyWorkerMain");
-	else
-		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
-
-	if (OidIsValid(relid))
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
-				 "logical replication worker for subscription %u sync %u", subid, relid);
-	else if (is_parallel_apply_worker)
+				 "logical replication parallel apply worker for subscription %u", subid);
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
 				 "logical replication parallel apply worker for subscription %u", subid);
+	}
+	else if (OidIsValid(relid))
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "TablesyncWorkerMain");
+		snprintf(bgw.bgw_name, BGW_MAXLEN,
+				 "logical replication tablesync worker for subscription %u sync %u", subid, relid);
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication tablesync worker");
+	}
 	else
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
 				 "logical replication apply worker for subscription %u", subid);
-
-	if (is_parallel_apply_worker)
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication parallel worker");
-	else
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker");
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication apply worker");
+	}
 
 	bgw.bgw_restart_time = BGW_NEVER_RESTART;
 	bgw.bgw_notify_pid = MyProcPid;
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 6d461654ab..8125bbd170 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -131,7 +131,7 @@ static StringInfo copybuf = NULL;
 /*
  * Exit routine for synchronization worker.
  */
-static void
+void
 pg_attribute_noreturn()
 finish_sync_worker(void)
 {
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 0ee764d68f..b979a755ae 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -392,6 +392,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void set_stream_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -4330,6 +4331,69 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* set_stream_options
+  * 	Set logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+set_stream_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+		options->proto.logical.twophase = true;
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4442,13 +4506,134 @@ start_apply(XLogRecPtr origin_startpos)
 }
 
 /*
- * Common initialization for leader apply worker and parallel apply worker.
+ * Runs the tablesync worker.
+ * It starts syncing tables. After a successful sync,
+ * sets streaming options and starts streaming to catchup.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Start table synchronization. */
+	start_table_sync(origin_startpos, &slotname);
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	/* Start applying changes to catchup. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Runs the leader apply worker.
+ * It sets up replication origin, streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+	bool		must_use_password;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard
+	 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+	 * crash if slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+
+	/* Is the use of a password mandatory? */
+	must_use_password = MySubscription->passwordrequired &&
+		!superuser_arg(MySubscription->owner);
+
+	/* Note that the superuser_arg call can access the DB */
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											must_use_password,
+											MySubscription->name, &err);
+
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it
+	 * does some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+
+	/* Run the main loop. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Common initialization for logical replication workers; leader apply worker,
+ * parallel apply worker and tablesync worker.
  *
  * Initialize the database connection, in-memory subscription and necessary
  * config options.
  */
 void
-InitializeApplyWorker(void)
+InitializeLogRepWorker(void)
 {
 	MemoryContext oldctx;
 
@@ -4512,7 +4697,8 @@ InitializeApplyWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+						get_worker_name(),
 						MySubscription->name,
 						get_rel_name(MyLogicalRepWorker->relid))));
 	else
@@ -4533,7 +4719,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	InitializingApplyWorker = true;
 
@@ -4557,7 +4742,7 @@ ApplyWorkerMain(Datum main_arg)
 	/* Load the libpq-specific functions */
 	load_file("libpqwalreceiver", false);
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
@@ -4565,165 +4750,69 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-		bool		must_use_password;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-
-		/* Is the use of a password mandatory? */
-		must_use_password = MySubscription->passwordrequired &&
-			!superuser_arg(MySubscription->owner);
-
-		/* Note that the superuser_arg call can access the DB */
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												must_use_password,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
+	/* This is leader apply worker */
+	run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
+	proc_exit(0);
+}
 
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
+/* Logical Replication Tablesync worker entry point */
+void
+TablesyncWorkerMain(Datum main_arg)
+{
+	int			worker_slot = DatumGetInt32(main_arg);
+	char		originname[NAMEDATALEN];
+	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
+	char	   *myslotname = NULL;
+	WalRcvStreamOptions options;
+
+	/* Attach to slot */
+	logicalrep_worker_attach(worker_slot);
+
+	/* Setup signal handling */
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
 
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * We don't currently need any ResourceOwner in a walreceiver process, but
+	 * if we did, we could call CreateAuxProcessResourceOwner here.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
 
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
+	/* Initialise stats to a sanish value */
+	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
+		MyLogicalRepWorker->reply_time = GetCurrentTimestamp();
 
-	if (!am_tablesync_worker())
-	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
-		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+	/* Load the libpq-specific functions */
+	load_file("libpqwalreceiver", false);
 
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
-		}
-		else
-		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-		}
+	InitializeLogRepWorker();
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-								 MySubscription->name,
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-								 "?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
+	/* Connect to the origin and start the replication. */
+	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
+		 MySubscription->conninfo);
 
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+	/*
+	 * Setup callback for syscache so that we know when something changes in
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
+	 */
+	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
+								  invalidate_syncing_table_states,
+								  (Datum) 0);
 
-	proc_exit(0);
+	run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+
+	finish_sync_worker();
 }
 
 /*
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index 39588da79f..bbd71d0b42 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -18,6 +18,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending;
 
 extern void ApplyWorkerMain(Datum main_arg);
 extern void ParallelApplyWorkerMain(Datum main_arg);
+extern void TablesyncWorkerMain(Datum main_arg);
 
 extern bool IsLogicalWorker(void);
 extern bool IsLogicalParallelApplyWorker(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 343e781896..7aba034774 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -265,7 +265,7 @@ extern void maybe_reread_subscription(void);
 
 extern void stream_cleanup_files(Oid subid, TransactionId xid);
 
-extern void InitializeApplyWorker(void);
+extern void InitializeLogRepWorker(void);
 
 extern void store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn);
 
@@ -307,6 +307,8 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
+extern void finish_sync_worker(void);
+
 static inline bool
 am_tablesync_worker(void)
 {
-- 
2.27.0

From 9f97e6dd29264e48581c6480a411f2f1948c5da3 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Mon, 5 Jun 2023 15:45:29 +0300
Subject: [PATCH 2/5] Reuse Tablesync Workers

This commit allows reusing tablesync workers for syncing more than one
table sequentially during their lifetime, instead of exiting after
only syncing one table.

Before this commit, tablesync workers were capable of syncing only one
table. For each table, a new sync worker was launched and that worker would
exit when done processing the table.

Now, tablesync workers are not limited to processing only one
table. When done, they can move to processing another table in
the same subscription.

If there is a table that needs to be synced, an available tablesync
worker picks up that table and syncs it. Each tablesync worker
continues to pick new tables to sync until there are no tables left
requiring synchronization. If there was no available worker to
process the table, then a new tablesync worker will be launched,
provided the number of tablesync workers for the subscription does not
exceed max_sync_workers_per_subscription.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/replication/logical/launcher.c  |   1 +
 src/backend/replication/logical/tablesync.c |  46 ++++++--
 src/backend/replication/logical/worker.c    | 110 +++++++++++++++++++-
 src/include/replication/worker_internal.h   |   7 ++
 4 files changed, 149 insertions(+), 15 deletions(-)

diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index f562b1db41..c24e56a11e 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -440,6 +440,7 @@ retry:
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
+	worker->is_sync_completed = false;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
 	TIMESTAMP_NOBEGIN(worker->last_recv_time);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8125bbd170..37f073b968 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -129,11 +129,10 @@ static bool FetchTableStates(bool *started_tx);
 static StringInfo copybuf = NULL;
 
 /*
- * Exit routine for synchronization worker.
+ * Prepares the synchronization worker for reuse or exit.
  */
 void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -145,19 +144,38 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	if (LogRepWorkerWalRcvConn != NULL)
+	{
+		walrcv_disconnect(LogRepWorkerWalRcvConn);
+		LogRepWorkerWalRcvConn = NULL;
+	}
+
+	/* Find the leader apply worker and signal it. */
+	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
 	/* And flush all writes. */
 	XLogFlush(GetXLogWriteRecPtr());
 
 	StartTransactionCommand();
 	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
+			(errmsg("logical replication table synchronization worker for subscription \"%s\" has finished",
+					MySubscription->name)));
 	CommitTransactionCommand();
 
-	/* Find the leader apply worker and signal it. */
-	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
-
 	/* Stop gracefully */
 	proc_exit(0);
 }
@@ -379,7 +397,15 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 */
 		replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+		/* Sync worker has completed synchronization of the current table. */
+		MyLogicalRepWorker->is_sync_completed = true;
+
+		ereport(LOG,
+		(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has finished",
+				MySubscription->name,
+				get_rel_name(MyLogicalRepWorker->relid),
+				MyLogicalRepWorker->relid)));
+		CommitTransactionCommand();
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index b979a755ae..4a0f402ad4 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3624,6 +3624,20 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can call process_syncing_tables_for_sync.
+				 *
+				 * process_syncing_tables_for_sync decides whether the sync of the
+				 * current table is completed. If it is completed, streaming must
+				 * be already ended. So, we can break the loop.
+				 */
+				if (MyLogicalRepWorker->is_sync_completed)
+				{
+					endofstream = true;
+					break;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3643,6 +3657,15 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+
+			/*
+			 * If is_sync_completed is true, this means that the tablesync worker
+			 * is done with synchronization. Streaming has already been ended by
+			 * process_syncing_tables_for_sync. We should move to the next table
+			 * if needed, or exit.
+			 */
+			if (MyLogicalRepWorker->is_sync_completed)
+				endofstream = true;
 		}
 
 		/* Cleanup the memory. */
@@ -3745,8 +3768,11 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/* Tablesync workers should end streaming before exiting the main loop
+	 * to drop replication slot. Only end streaming here for apply workers.
+	 */
+	if (!am_tablesync_worker())
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
 }
 
 /*
@@ -4517,6 +4543,8 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 					 int originname_size,
 					 XLogRecPtr *origin_startpos)
 {
+	MyLogicalRepWorker->is_sync_completed = false;
+
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname);
 
@@ -4697,10 +4725,11 @@ InitializeLogRepWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" with relid %u has started",
 						get_worker_name(),
 						MySubscription->name,
-						get_rel_name(MyLogicalRepWorker->relid))));
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
 	else
 		ereport(LOG,
 		/* translator: first %s is the name of logical replication worker */
@@ -4810,7 +4839,78 @@ TablesyncWorkerMain(Datum main_arg)
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+	/*
+	 * The loop where worker does its job. It loops until there is no relation
+	 * left to sync.
+	 */
+	for (;;)
+	{
+		List	   *rstates;
+		SubscriptionRelState *rstate;
+		ListCell   *lc;
+		bool 	is_table_found = false;
+
+		run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+
+		if (IsTransactionState())
+			CommitTransactionCommand();
+
+		if (MyLogicalRepWorker->is_sync_completed)
+		{
+			/* This transaction will be committed by clean_sync_worker. */
+			StartTransactionCommand();
+
+			/*
+			 * Check if there is any table whose relation state is still INIT.
+			 * If a table in INIT state is found, the worker will not be finished,
+			 * it will be reused instead.
+			 */
+			rstates = GetSubscriptionRelations(MySubscription->oid, true);
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+
+			foreach(lc, rstates)
+			{
+				memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+				/*
+				* Pick the table for the next run if it is not already picked up
+				* by another worker.
+				*
+				* Take exclusive lock to prevent any other sync worker from picking
+				* the same table.
+				*/
+				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
+				if (rstate->state != SUBREL_STATE_SYNCDONE &&
+					!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+				{
+					/* Update worker state for the next table */
+					MyLogicalRepWorker->relid = rstate->relid;
+					MyLogicalRepWorker->relstate = rstate->state;
+					MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+					LWLockRelease(LogicalRepWorkerLock);
+
+					/* found a table for next iteration */
+					is_table_found = true;
+					clean_sync_worker();
+
+					StartTransactionCommand();
+					ereport(LOG,
+							(errmsg("%s for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+									get_worker_name(),
+									MySubscription->name,
+									get_rel_name(MyLogicalRepWorker->relid),
+									MyLogicalRepWorker->relid)));
+					CommitTransactionCommand();
+
+					break;
+				}
+				LWLockRelease(LogicalRepWorkerLock);
+			}
+
+			if (!is_table_found)
+				break;
+		}
+	}
 
 	finish_sync_worker();
 }
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 7aba034774..1e9f8e6e72 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -56,6 +56,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Indicates whether tablesync worker has completed sycning its assigned
+	 * table. If true, no need to continue with that table.
+	 */
+	bool		is_sync_completed;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -308,6 +314,7 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
 extern void finish_sync_worker(void);
+extern void clean_sync_worker(void);
 
 static inline bool
 am_tablesync_worker(void)
-- 
2.27.0

From 3ff63e4db6966194b03c86e77c1b312aab5e26cf Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <kuroda.hayato@fujitsu.com>
Date: Tue, 27 Jun 2023 07:10:45 +0000
Subject: [PATCH 3/5] reuse connection when tablesync workers change the target

---
 src/backend/replication/logical/tablesync.c | 21 ++++++++++++++-------
 src/backend/replication/logical/worker.c    | 20 ++++++++++++--------
 src/backend/replication/walsender.c         |  6 ++++++
 3 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 37f073b968..30fe9b78ac 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -1321,14 +1321,21 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 									NAMEDATALEN);
 
 	/*
-	 * Here we use the slot name instead of the subscription name as the
-	 * application_name, so that it is different from the leader apply worker,
-	 * so that synchronous replication can distinguish them.
+	 * Connect to publisher if not yet. The application_name must be also
+	 * different from the leader apply worker.
 	 */
-	LogRepWorkerWalRcvConn =
-		walrcv_connect(MySubscription->conninfo, true,
-					   must_use_password,
-					   slotname, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+	{
+		char application_name[NAMEDATALEN];
+
+		snprintf(application_name, NAMEDATALEN, "tablesync for %s",
+				 MySubscription->name);
+		LogRepWorkerWalRcvConn =
+			walrcv_connect(MySubscription->conninfo, true,
+						   must_use_password,
+						   application_name, &err);
+	}
+
 	if (LogRepWorkerWalRcvConn == NULL)
 		ereport(ERROR,
 				(errcode(ERRCODE_CONNECTION_FAILURE),
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 4a0f402ad4..8df960e343 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3498,19 +3498,21 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 	/*
 	 * Init the ApplyMessageContext which we clean up after each replication
-	 * protocol message.
+	 * protocol message, if needed.
 	 */
-	ApplyMessageContext = AllocSetContextCreate(ApplyContext,
-												"ApplyMessageContext",
-												ALLOCSET_DEFAULT_SIZES);
+	if (!ApplyMessageContext)
+		ApplyMessageContext = AllocSetContextCreate(ApplyContext,
+													"ApplyMessageContext",
+													ALLOCSET_DEFAULT_SIZES);
 
 	/*
 	 * This memory context is used for per-stream data when the streaming mode
 	 * is enabled. This context is reset on each stream stop.
 	 */
-	LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
-													"LogicalStreamingContext",
-													ALLOCSET_DEFAULT_SIZES);
+	if (!LogicalStreamingContext)
+		LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
+														"LogicalStreamingContext",
+														ALLOCSET_DEFAULT_SIZES);
 
 	/* mark as idle, before starting to loop */
 	pgstat_report_activity(STATE_IDLE, NULL);
@@ -4891,7 +4893,9 @@ TablesyncWorkerMain(Datum main_arg)
 
 					/* found a table for next iteration */
 					is_table_found = true;
-					clean_sync_worker();
+
+					CommitTransactionCommand();
+					pgstat_report_stat(true);
 
 					StartTransactionCommand();
 					ereport(LOG,
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index d3a136b6f5..429d00f2f0 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1828,6 +1828,12 @@ exec_replication_command(const char *cmd_string)
 				set_ps_display(cmdtag);
 				PreventInTransactionBlock(true, cmdtag);
 
+				/*
+				 * Initialize the flag again because this streaming may be
+				 * second time.
+				 */
+				streamingDoneSending = streamingDoneReceiving = false;
+
 				if (cmd->kind == REPLICATION_KIND_PHYSICAL)
 					StartReplication(cmd);
 				else
-- 
2.27.0

From 0092e4a0b5e67416e0bd0ed55d5dd5bbb001c7a8 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 5/6] Reuse Replication Slot and Origin in Tablesync

This commit allows reusing replication slots and origins during tablesync.

Earlier, a tablesync worker was creating a new replication slot and origin
each time it syncs a new table. With this patch, replication
slots/origins can be reusable for tablesync.

This reduces the overhead of creating/dropping replication slots and origins
and improves tablesync speed significantly especially for empty or small tables.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create those first. Otherwise the worker reuses
slot and origin created by the same worker in previous iterations earlier.
Tables in FINISHEDCOPY are expected to have a replication slot and origin.
Slot and origin names for such tables are persisted in
pg_subscription_rel catalog. Tablesync worker can fetch them and proceed
with existing slot and origin of FINISHEDCOPY tables and does not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 247 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 229 +++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   8 +-
 src/backend/replication/logical/tablesync.c   | 369 +++++++++++++-----
 src/backend/replication/logical/worker.c      |  62 ++-
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  14 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  23 +-
 11 files changed, 797 insertions(+), 198 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index ed32ca0349..0183ecd21e 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8030,6 +8030,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8114,6 +8127,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index d07f88ce28..152fdaa310 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -108,6 +108,14 @@ GetSubscription(Oid subid, bool missing_ok)
 								   Anum_pg_subscription_suborigin);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -199,6 +207,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -228,7 +274,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -257,6 +303,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -269,6 +325,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -299,14 +409,56 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -318,6 +470,85 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 54895ba929..9f5a295ddc 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -710,6 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -773,7 +774,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -864,6 +865,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
 	bool		must_use_password;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -943,7 +946,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -967,6 +970,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -993,13 +997,36 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
 				 * tablesync origin.
 				 */
-				if (state != SUBREL_STATE_READY)
+				if (state != SUBREL_STATE_READY &&
+					state != SUBREL_STATE_SYNCDONE)
 				{
 					char		originname[NAMEDATALEN];
 
@@ -1027,31 +1054,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1474,6 +1494,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1546,6 +1567,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1595,6 +1624,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1606,36 +1637,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1688,39 +1709,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1743,6 +1742,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2155,6 +2219,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2165,18 +2230,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index f16e2377bf..a57f178087 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										InvalidRepSlotId);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index c24e56a11e..d0d8d95a68 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -1180,7 +1183,8 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID,
+										 InvalidRepSlotId);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 0fc81355bf..4c81a9b4aa 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -328,40 +328,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication tracking origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -370,32 +359,70 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
-		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/*
+			 * Drop replication origin
+			 *
+			 * There is a chance that the user is concurrently performing refresh
+			 * for the subscription where we remove the table state and its origin
+			 * or the apply worker would have removed this origin. So passing
+			 * missing_ok = true.
+			 */
+			replorigin_drop_by_name(originname, true, false);
+		}
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+		ereport(DEBUG2,
+			(errmsg("process_syncing_tables_for_sync: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					"NULL", "NULL", MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
 
 		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		StartTransactionCommand();
+		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+										   MyLogicalRepWorker->relid,
+										   originname,
+										   sizeof(originname));
 
 		/* Sync worker has completed synchronization of the current table. */
 		MyLogicalRepWorker->is_sync_completed = true;
@@ -492,6 +519,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -512,18 +540,31 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+				ereport(DEBUG2,
+					(errmsg("process_syncing_tables_for_apply: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+							"NULL", "NULL", rstate->state,
+							rstate->relid, MyLogicalRepWorker->subid)));
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -612,12 +653,25 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+						ereport(DEBUG2,
+								(errmsg("process_syncing_tables_for_apply: incremented lastusedid to %lld for subscription %u",
+										(long long) MySubscription->lastusedid, MySubscription->oid)));
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1240,8 +1294,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1252,11 +1306,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1282,6 +1336,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	UserContext ucxt;
 	bool		must_use_password;
 	bool		run_as_owner;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1316,7 +1371,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1355,12 +1410,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1374,10 +1443,53 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+			ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL", "NULL", MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		memcpy(slotname, prev_slotname, NAMEDATALEN);
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1397,7 +1509,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1405,11 +1519,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+
+	/*
+	 * Refresh the originname in case of having non-existing origin
+	 * from previous failed sync attempts.
+	 * If that's the case, it should be removed from the catalog so far.
+	 * Then, we can continue by reusing the origin created by the current
+	 * worker instead of .
+	 */
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									MyLogicalRepWorker->relid,
+									originname,
+									sizeof(originname));
+
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					slotname, originname, MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
 	pgstat_report_stat(true);
 
 	StartTransactionCommand();
@@ -1437,48 +1571,95 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: created replication slot %s for subscription %u",
+						slotname, MyLogicalRepWorker->subid)));
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: reusing replication slot %s for relation %u in subscription %u",
+						slotname, MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/*
 	 * Make sure that the copy command runs as the table owner, unless the
 	 * user has opted out of that behaviour.
@@ -1537,12 +1718,18 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					originname, slotname, SUBREL_STATE_FINISHEDCOPY,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 
 copy_table_done:
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 8df960e343..c8f8b72aa7 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -464,8 +464,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -3770,7 +3778,8 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* Tablesync workers should end streaming before exiting the main loop
+	/*
+	 * Tablesync workers should end streaming before exiting the main loop
 	 * to drop replication slot. Only end streaming here for apply workers.
 	 */
 	if (!am_tablesync_worker())
@@ -4496,6 +4505,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4550,10 +4562,12 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname);
 
+	StartTransactionCommand();
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   originname_size);
+	CommitTransactionCommand();
 
 	set_apply_error_context_origin(originname);
 
@@ -4594,11 +4608,10 @@ run_apply_worker(WalRcvStreamOptions *options,
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 					errmsg("subscription has no replication slot set")));
 
-	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-									   originname, originname_size);
-
 	/* Setup replication origin tracking. */
 	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
 	originid = replorigin_by_name(originname, true);
 	if (!OidIsValid(originid))
 		originid = replorigin_create(originname);
@@ -4875,12 +4888,12 @@ TablesyncWorkerMain(Datum main_arg)
 				memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
 
 				/*
-				* Pick the table for the next run if it is not already picked up
-				* by another worker.
-				*
-				* Take exclusive lock to prevent any other sync worker from picking
-				* the same table.
-				*/
+				 * Pick the table for the next run if it is not already picked up
+				 * by another worker.
+				 *
+				 * Take exclusive lock to prevent any other sync worker from picking
+				 * the same table.
+				 */
 				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
 				if (rstate->state != SUBREL_STATE_SYNCDONE &&
 					!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
@@ -4912,7 +4925,32 @@ TablesyncWorkerMain(Datum main_arg)
 			}
 
 			if (!is_table_found)
+			{
+				TimeLineID	tli;
+
+				/*
+				 * It is important to give an error if we are unable to drop the
+				 * slot, otherwise, it won't be dropped till the corresponding
+				 * subscription is dropped. So passing missing_ok = false.
+				 */
+				if (MyLogicalRepWorker->created_slot)
+				{
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+					ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+				}
+
+				/*
+				 * Drop replication origin before exiting.
+				 *
+				 * There is a chance that the user is concurrently performing refresh
+				 * for the subscription where we remove the table state and its origin
+				 * or the apply worker would have removed this origin. So passing
+				 * missing_ok = true.
+				 */
+				replorigin_drop_by_name(originname, true, false);
+
 				break;
+			}
 		}
 	}
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 1d40eebc78..7e13f59847 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -108,6 +108,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -144,6 +147,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -164,6 +169,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..185164d75e 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,16 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index a8a89dc784..31b2c41893 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -237,8 +237,9 @@ extern bool InvalidateObsoleteReplicationSlots(ReplicationSlotInvalidationCause
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 1e9f8e6e72..24006240c9 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -239,7 +256,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
@@ -335,4 +353,7 @@ am_parallel_apply_worker(void)
 	return isParallelApplyWorker(MyLogicalRepWorker);
 }
 
+/* Invalid identifier to be used for naming replication slots */
+#define InvalidRepSlotId	0
+
 #endif							/* WORKER_INTERNAL_H */
-- 
2.27.0

From c1c5c8bf5284a397b3a3efdc83be69285e98baa2 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <kuroda.hayato@fujitsu.com>
Date: Wed, 28 Jun 2023 06:19:34 +0000
Subject: [PATCH 6/6] Use slot name as application_name again

---
 src/backend/replication/logical/tablesync.c | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 4c81a9b4aa..da0264c0b6 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -1382,23 +1382,14 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 	 */
 	if (LogRepWorkerWalRcvConn == NULL)
 	{
-		char application_name[NAMEDATALEN];
-
 		/*
-		 * FIXME: set appropriate application_name. Previously, the slot name
-		 * was used becasue the lifetime of the tablesync worker was same as
-		 * that, but now the tablesync worker handles many slots during the
-		 * synchronization so that it is not suitable. So what should be?
-		 * Note that if the tablesync worker starts to reuse the replication
-		 * slot during synchronization, we should use the slot name as
-		 * application_name again.
+		 * Here we use the slot name instead of the subscription name as the
+		 * application_name,
 		 */
-		snprintf(application_name, NAMEDATALEN, "tablesync for %s",
-				 MySubscription->name);
 		LogRepWorkerWalRcvConn =
 			walrcv_connect(MySubscription->conninfo, true,
 						   must_use_password,
-						   application_name, &err);
+						   slotname, &err);
 	}
 
 	if (LogRepWorkerWalRcvConn == NULL)
-- 
2.27.0

From 46942df115ce0e0d653b932c61d9b4c7cfab7807 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Mon, 5 Jun 2023 15:04:41 +0300
Subject: [PATCH 1/6] Refactor to split Apply and Tablesync Workers

Both apply and tablesync workers were using ApplyWorkerMain() as entry
point. As the name implies, ApplyWorkerMain() should be considered as
the main function for apply workers. Tablesync worker's path was hidden
and does not have enough in common to share the same main function with
apply worker.

Also; most of the code shared by both worker types are already combined
in LogicalRepApplyLoop(). There is no need to combine the rest in
ApplyWorkerMain() anymore.

This commit introduces TablesyncWorkerMain() as a new entry point for
tablesync workers and separates both type of workers from each other.
This aims to increase code readability and help to maintain logical
replication workers separately.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/postmaster/bgworker.c             |   3 +
 .../replication/logical/applyparallelworker.c |   2 +-
 src/backend/replication/logical/launcher.c    |  25 +-
 src/backend/replication/logical/tablesync.c   |   2 +-
 src/backend/replication/logical/worker.c      | 381 +++++++++++-------
 src/include/replication/logicalworker.h       |   1 +
 src/include/replication/worker_internal.h     |   4 +-
 7 files changed, 258 insertions(+), 160 deletions(-)

diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 0dd22b2351..5609919edf 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -131,6 +131,9 @@ static const struct
 	},
 	{
 		"ParallelApplyWorkerMain", ParallelApplyWorkerMain
+	},
+	{
+		"TablesyncWorkerMain", TablesyncWorkerMain
 	}
 };
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 82c1ddcdcb..f16e2377bf 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -942,7 +942,7 @@ ParallelApplyWorkerMain(Datum main_arg)
 	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
 		MyLogicalRepWorker->reply_time = 0;
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 8395ae7b23..f562b1db41 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -459,24 +459,27 @@ retry:
 	snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres");
 
 	if (is_parallel_apply_worker)
+	{
 		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ParallelApplyWorkerMain");
-	else
-		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
-
-	if (OidIsValid(relid))
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
-				 "logical replication worker for subscription %u sync %u", subid, relid);
-	else if (is_parallel_apply_worker)
+				 "logical replication parallel apply worker for subscription %u", subid);
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
 				 "logical replication parallel apply worker for subscription %u", subid);
+	}
+	else if (OidIsValid(relid))
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "TablesyncWorkerMain");
+		snprintf(bgw.bgw_name, BGW_MAXLEN,
+				 "logical replication tablesync worker for subscription %u sync %u", subid, relid);
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication tablesync worker");
+	}
 	else
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
 				 "logical replication apply worker for subscription %u", subid);
-
-	if (is_parallel_apply_worker)
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication parallel worker");
-	else
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker");
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication apply worker");
+	}
 
 	bgw.bgw_restart_time = BGW_NEVER_RESTART;
 	bgw.bgw_notify_pid = MyProcPid;
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 6d461654ab..8125bbd170 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -131,7 +131,7 @@ static StringInfo copybuf = NULL;
 /*
  * Exit routine for synchronization worker.
  */
-static void
+void
 pg_attribute_noreturn()
 finish_sync_worker(void)
 {
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 0ee764d68f..b979a755ae 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -392,6 +392,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void set_stream_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -4330,6 +4331,69 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* set_stream_options
+  * 	Set logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+set_stream_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+		options->proto.logical.twophase = true;
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4442,13 +4506,134 @@ start_apply(XLogRecPtr origin_startpos)
 }
 
 /*
- * Common initialization for leader apply worker and parallel apply worker.
+ * Runs the tablesync worker.
+ * It starts syncing tables. After a successful sync,
+ * sets streaming options and starts streaming to catchup.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Start table synchronization. */
+	start_table_sync(origin_startpos, &slotname);
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	/* Start applying changes to catchup. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Runs the leader apply worker.
+ * It sets up replication origin, streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+	bool		must_use_password;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard
+	 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+	 * crash if slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+
+	/* Is the use of a password mandatory? */
+	must_use_password = MySubscription->passwordrequired &&
+		!superuser_arg(MySubscription->owner);
+
+	/* Note that the superuser_arg call can access the DB */
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											must_use_password,
+											MySubscription->name, &err);
+
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it
+	 * does some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+
+	/* Run the main loop. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Common initialization for logical replication workers; leader apply worker,
+ * parallel apply worker and tablesync worker.
  *
  * Initialize the database connection, in-memory subscription and necessary
  * config options.
  */
 void
-InitializeApplyWorker(void)
+InitializeLogRepWorker(void)
 {
 	MemoryContext oldctx;
 
@@ -4512,7 +4697,8 @@ InitializeApplyWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+						get_worker_name(),
 						MySubscription->name,
 						get_rel_name(MyLogicalRepWorker->relid))));
 	else
@@ -4533,7 +4719,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	InitializingApplyWorker = true;
 
@@ -4557,7 +4742,7 @@ ApplyWorkerMain(Datum main_arg)
 	/* Load the libpq-specific functions */
 	load_file("libpqwalreceiver", false);
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
@@ -4565,165 +4750,69 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-		bool		must_use_password;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-
-		/* Is the use of a password mandatory? */
-		must_use_password = MySubscription->passwordrequired &&
-			!superuser_arg(MySubscription->owner);
-
-		/* Note that the superuser_arg call can access the DB */
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												must_use_password,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
+	/* This is leader apply worker */
+	run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
+	proc_exit(0);
+}
 
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
+/* Logical Replication Tablesync worker entry point */
+void
+TablesyncWorkerMain(Datum main_arg)
+{
+	int			worker_slot = DatumGetInt32(main_arg);
+	char		originname[NAMEDATALEN];
+	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
+	char	   *myslotname = NULL;
+	WalRcvStreamOptions options;
+
+	/* Attach to slot */
+	logicalrep_worker_attach(worker_slot);
+
+	/* Setup signal handling */
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
 
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * We don't currently need any ResourceOwner in a walreceiver process, but
+	 * if we did, we could call CreateAuxProcessResourceOwner here.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
 
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
+	/* Initialise stats to a sanish value */
+	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
+		MyLogicalRepWorker->reply_time = GetCurrentTimestamp();
 
-	if (!am_tablesync_worker())
-	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
-		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+	/* Load the libpq-specific functions */
+	load_file("libpqwalreceiver", false);
 
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
-		}
-		else
-		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-		}
+	InitializeLogRepWorker();
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-								 MySubscription->name,
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-								 "?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
+	/* Connect to the origin and start the replication. */
+	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
+		 MySubscription->conninfo);
 
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+	/*
+	 * Setup callback for syscache so that we know when something changes in
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
+	 */
+	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
+								  invalidate_syncing_table_states,
+								  (Datum) 0);
 
-	proc_exit(0);
+	run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+
+	finish_sync_worker();
 }
 
 /*
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index 39588da79f..bbd71d0b42 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -18,6 +18,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending;
 
 extern void ApplyWorkerMain(Datum main_arg);
 extern void ParallelApplyWorkerMain(Datum main_arg);
+extern void TablesyncWorkerMain(Datum main_arg);
 
 extern bool IsLogicalWorker(void);
 extern bool IsLogicalParallelApplyWorker(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 343e781896..7aba034774 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -265,7 +265,7 @@ extern void maybe_reread_subscription(void);
 
 extern void stream_cleanup_files(Oid subid, TransactionId xid);
 
-extern void InitializeApplyWorker(void);
+extern void InitializeLogRepWorker(void);
 
 extern void store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn);
 
@@ -307,6 +307,8 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
+extern void finish_sync_worker(void);
+
 static inline bool
 am_tablesync_worker(void)
 {
-- 
2.27.0

From f04d0fd0099d8c3a3727341ecb868031fb9f93b3 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Mon, 5 Jun 2023 15:45:29 +0300
Subject: [PATCH 2/6] Reuse Tablesync Workers

This commit allows reusing tablesync workers for syncing more than one
table sequentially during their lifetime, instead of exiting after
only syncing one table.

Before this commit, tablesync workers were capable of syncing only one
table. For each table, a new sync worker was launched and that worker would
exit when done processing the table.

Now, tablesync workers are not limited to processing only one
table. When done, they can move to processing another table in
the same subscription.

If there is a table that needs to be synced, an available tablesync
worker picks up that table and syncs it. Each tablesync worker
continues to pick new tables to sync until there are no tables left
requiring synchronization. If there was no available worker to
process the table, then a new tablesync worker will be launched,
provided the number of tablesync workers for the subscription does not
exceed max_sync_workers_per_subscription.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/replication/logical/launcher.c  |   1 +
 src/backend/replication/logical/tablesync.c |  46 ++++++--
 src/backend/replication/logical/worker.c    | 110 +++++++++++++++++++-
 src/include/replication/worker_internal.h   |   7 ++
 4 files changed, 149 insertions(+), 15 deletions(-)

diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index f562b1db41..c24e56a11e 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -440,6 +440,7 @@ retry:
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
+	worker->is_sync_completed = false;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
 	TIMESTAMP_NOBEGIN(worker->last_recv_time);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8125bbd170..37f073b968 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -129,11 +129,10 @@ static bool FetchTableStates(bool *started_tx);
 static StringInfo copybuf = NULL;
 
 /*
- * Exit routine for synchronization worker.
+ * Prepares the synchronization worker for reuse or exit.
  */
 void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -145,19 +144,38 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	if (LogRepWorkerWalRcvConn != NULL)
+	{
+		walrcv_disconnect(LogRepWorkerWalRcvConn);
+		LogRepWorkerWalRcvConn = NULL;
+	}
+
+	/* Find the leader apply worker and signal it. */
+	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
 	/* And flush all writes. */
 	XLogFlush(GetXLogWriteRecPtr());
 
 	StartTransactionCommand();
 	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
+			(errmsg("logical replication table synchronization worker for subscription \"%s\" has finished",
+					MySubscription->name)));
 	CommitTransactionCommand();
 
-	/* Find the leader apply worker and signal it. */
-	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
-
 	/* Stop gracefully */
 	proc_exit(0);
 }
@@ -379,7 +397,15 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 */
 		replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+		/* Sync worker has completed synchronization of the current table. */
+		MyLogicalRepWorker->is_sync_completed = true;
+
+		ereport(LOG,
+		(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has finished",
+				MySubscription->name,
+				get_rel_name(MyLogicalRepWorker->relid),
+				MyLogicalRepWorker->relid)));
+		CommitTransactionCommand();
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index b979a755ae..4a0f402ad4 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3624,6 +3624,20 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can call process_syncing_tables_for_sync.
+				 *
+				 * process_syncing_tables_for_sync decides whether the sync of the
+				 * current table is completed. If it is completed, streaming must
+				 * be already ended. So, we can break the loop.
+				 */
+				if (MyLogicalRepWorker->is_sync_completed)
+				{
+					endofstream = true;
+					break;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3643,6 +3657,15 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+
+			/*
+			 * If is_sync_completed is true, this means that the tablesync worker
+			 * is done with synchronization. Streaming has already been ended by
+			 * process_syncing_tables_for_sync. We should move to the next table
+			 * if needed, or exit.
+			 */
+			if (MyLogicalRepWorker->is_sync_completed)
+				endofstream = true;
 		}
 
 		/* Cleanup the memory. */
@@ -3745,8 +3768,11 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/* Tablesync workers should end streaming before exiting the main loop
+	 * to drop replication slot. Only end streaming here for apply workers.
+	 */
+	if (!am_tablesync_worker())
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
 }
 
 /*
@@ -4517,6 +4543,8 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 					 int originname_size,
 					 XLogRecPtr *origin_startpos)
 {
+	MyLogicalRepWorker->is_sync_completed = false;
+
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname);
 
@@ -4697,10 +4725,11 @@ InitializeLogRepWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" with relid %u has started",
 						get_worker_name(),
 						MySubscription->name,
-						get_rel_name(MyLogicalRepWorker->relid))));
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
 	else
 		ereport(LOG,
 		/* translator: first %s is the name of logical replication worker */
@@ -4810,7 +4839,78 @@ TablesyncWorkerMain(Datum main_arg)
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+	/*
+	 * The loop where worker does its job. It loops until there is no relation
+	 * left to sync.
+	 */
+	for (;;)
+	{
+		List	   *rstates;
+		SubscriptionRelState *rstate;
+		ListCell   *lc;
+		bool 	is_table_found = false;
+
+		run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+
+		if (IsTransactionState())
+			CommitTransactionCommand();
+
+		if (MyLogicalRepWorker->is_sync_completed)
+		{
+			/* This transaction will be committed by clean_sync_worker. */
+			StartTransactionCommand();
+
+			/*
+			 * Check if there is any table whose relation state is still INIT.
+			 * If a table in INIT state is found, the worker will not be finished,
+			 * it will be reused instead.
+			 */
+			rstates = GetSubscriptionRelations(MySubscription->oid, true);
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+
+			foreach(lc, rstates)
+			{
+				memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+				/*
+				* Pick the table for the next run if it is not already picked up
+				* by another worker.
+				*
+				* Take exclusive lock to prevent any other sync worker from picking
+				* the same table.
+				*/
+				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
+				if (rstate->state != SUBREL_STATE_SYNCDONE &&
+					!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+				{
+					/* Update worker state for the next table */
+					MyLogicalRepWorker->relid = rstate->relid;
+					MyLogicalRepWorker->relstate = rstate->state;
+					MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+					LWLockRelease(LogicalRepWorkerLock);
+
+					/* found a table for next iteration */
+					is_table_found = true;
+					clean_sync_worker();
+
+					StartTransactionCommand();
+					ereport(LOG,
+							(errmsg("%s for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+									get_worker_name(),
+									MySubscription->name,
+									get_rel_name(MyLogicalRepWorker->relid),
+									MyLogicalRepWorker->relid)));
+					CommitTransactionCommand();
+
+					break;
+				}
+				LWLockRelease(LogicalRepWorkerLock);
+			}
+
+			if (!is_table_found)
+				break;
+		}
+	}
 
 	finish_sync_worker();
 }
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 7aba034774..1e9f8e6e72 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -56,6 +56,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Indicates whether tablesync worker has completed sycning its assigned
+	 * table. If true, no need to continue with that table.
+	 */
+	bool		is_sync_completed;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -308,6 +314,7 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
 extern void finish_sync_worker(void);
+extern void clean_sync_worker(void);
 
 static inline bool
 am_tablesync_worker(void)
-- 
2.27.0

From f621571b4406b71e8522cc7846030cb3d11fccef Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <kuroda.hayato@fujitsu.com>
Date: Tue, 27 Jun 2023 07:10:45 +0000
Subject: [PATCH 3/6] reuse connection when tablesync workers change the target

---
 src/backend/replication/logical/tablesync.c | 31 ++++++++++++++++-----
 src/backend/replication/logical/worker.c    | 20 +++++++------
 src/backend/replication/walsender.c         |  6 ++++
 3 files changed, 42 insertions(+), 15 deletions(-)

diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 37f073b968..0fc81355bf 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -1321,14 +1321,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 									NAMEDATALEN);
 
 	/*
-	 * Here we use the slot name instead of the subscription name as the
-	 * application_name, so that it is different from the leader apply worker,
-	 * so that synchronous replication can distinguish them.
+	 * Connect to publisher if not yet. The application_name must be also
+	 * different from the leader apply worker because synchronous replication
+	 * must distinguish them.
 	 */
-	LogRepWorkerWalRcvConn =
-		walrcv_connect(MySubscription->conninfo, true,
-					   must_use_password,
-					   slotname, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+	{
+		char application_name[NAMEDATALEN];
+
+		/*
+		 * FIXME: set appropriate application_name. Previously, the slot name
+		 * was used becasue the lifetime of the tablesync worker was same as
+		 * that, but now the tablesync worker handles many slots during the
+		 * synchronization so that it is not suitable. So what should be?
+		 * Note that if the tablesync worker starts to reuse the replication
+		 * slot during synchronization, we should use the slot name as
+		 * application_name again.
+		 */
+		snprintf(application_name, NAMEDATALEN, "tablesync for %s",
+				 MySubscription->name);
+		LogRepWorkerWalRcvConn =
+			walrcv_connect(MySubscription->conninfo, true,
+						   must_use_password,
+						   application_name, &err);
+	}
+
 	if (LogRepWorkerWalRcvConn == NULL)
 		ereport(ERROR,
 				(errcode(ERRCODE_CONNECTION_FAILURE),
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 4a0f402ad4..8df960e343 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3498,19 +3498,21 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 	/*
 	 * Init the ApplyMessageContext which we clean up after each replication
-	 * protocol message.
+	 * protocol message, if needed.
 	 */
-	ApplyMessageContext = AllocSetContextCreate(ApplyContext,
-												"ApplyMessageContext",
-												ALLOCSET_DEFAULT_SIZES);
+	if (!ApplyMessageContext)
+		ApplyMessageContext = AllocSetContextCreate(ApplyContext,
+													"ApplyMessageContext",
+													ALLOCSET_DEFAULT_SIZES);
 
 	/*
 	 * This memory context is used for per-stream data when the streaming mode
 	 * is enabled. This context is reset on each stream stop.
 	 */
-	LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
-													"LogicalStreamingContext",
-													ALLOCSET_DEFAULT_SIZES);
+	if (!LogicalStreamingContext)
+		LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
+														"LogicalStreamingContext",
+														ALLOCSET_DEFAULT_SIZES);
 
 	/* mark as idle, before starting to loop */
 	pgstat_report_activity(STATE_IDLE, NULL);
@@ -4891,7 +4893,9 @@ TablesyncWorkerMain(Datum main_arg)
 
 					/* found a table for next iteration */
 					is_table_found = true;
-					clean_sync_worker();
+
+					CommitTransactionCommand();
+					pgstat_report_stat(true);
 
 					StartTransactionCommand();
 					ereport(LOG,
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index d3a136b6f5..429d00f2f0 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1828,6 +1828,12 @@ exec_replication_command(const char *cmd_string)
 				set_ps_display(cmdtag);
 				PreventInTransactionBlock(true, cmdtag);
 
+				/*
+				 * Initialize the flag again because this streaming may be
+				 * second time.
+				 */
+				streamingDoneSending = streamingDoneReceiving = false;
+
 				if (cmd->kind == REPLICATION_KIND_PHYSICAL)
 					StartReplication(cmd);
 				else
-- 
2.27.0

From 8c9494ccd472d710141f2aaf9b546658bb5486a7 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 4/6] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy
a relation and then apply changes by consuming from replication slot.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 ++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 104 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 289 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index b11d9a6ba3..f7d5acee93 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2595,6 +2595,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index dc9c5c82d9..cf88a3b7d4 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -82,6 +82,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -98,7 +100,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -982,6 +985,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 41243d0187..3c11f8e1dd 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -476,6 +476,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, must obtain a snapshot able to read all tables;
+ *  	if false, one that can read only catalogs is acceptable.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -494,6 +498,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -502,6 +507,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -571,8 +577,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 55a24c02c9..85c5cdb633 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 6035cf4816..c5b2d5b61f 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -486,6 +486,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 429d00f2f0..974ab5ca35 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1284,7 +1285,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1336,6 +1337,98 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	CheckLogicalDecodingRequirements();
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consistent WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1870,6 +1963,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 281626fa6f..f01e5f23a7 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -386,6 +386,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -403,6 +413,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -437,6 +448,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 260854747b..985a4ab7fd 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2348,6 +2348,7 @@ ReplicationSlotInvalidationCause
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3853,6 +3854,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.27.0

From 510ed4e64a6985f251ececc72f3b1a4312a3a185 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Mon, 5 Jun 2023 15:04:41 +0300
Subject: [PATCH v2 1/6] Refactor to split Apply and Tablesync Workers

Both apply and tablesync workers were using ApplyWorkerMain() as entry
point. As the name implies, ApplyWorkerMain() should be considered as
the main function for apply workers. Tablesync worker's path was hidden
and does not have enough in common to share the same main function with
apply worker.

Also; most of the code shared by both worker types are already combined
in LogicalRepApplyLoop(). There is no need to combine the rest in
ApplyWorkerMain() anymore.

This commit introduces TablesyncWorkerMain() as a new entry point for
tablesync workers and separates both type of workers from each other.
This aims to increase code readability and help to maintain logical
replication workers separately.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/postmaster/bgworker.c             |   3 +
 .../replication/logical/applyparallelworker.c |   2 +-
 src/backend/replication/logical/launcher.c    |  25 +-
 src/backend/replication/logical/tablesync.c   |   2 +-
 src/backend/replication/logical/worker.c      | 381 +++++++++++-------
 src/include/replication/logicalworker.h       |   1 +
 src/include/replication/worker_internal.h     |   4 +-
 7 files changed, 258 insertions(+), 160 deletions(-)

diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 5b4bd71694..505e38376c 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -131,6 +131,9 @@ static const struct
 	},
 	{
 		"ParallelApplyWorkerMain", ParallelApplyWorkerMain
+	},
+	{
+		"TablesyncWorkerMain", TablesyncWorkerMain
 	}
 };
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 6fb96148f4..1d4e83c4c1 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -942,7 +942,7 @@ ParallelApplyWorkerMain(Datum main_arg)
 	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
 		MyLogicalRepWorker->reply_time = 0;
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 542af7d863..c49a7fe811 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -459,24 +459,27 @@ retry:
 	snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres");
 
 	if (is_parallel_apply_worker)
+	{
 		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ParallelApplyWorkerMain");
-	else
-		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
-
-	if (OidIsValid(relid))
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
-				 "logical replication worker for subscription %u sync %u", subid, relid);
-	else if (is_parallel_apply_worker)
+				 "logical replication parallel apply worker for subscription %u", subid);
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
 				 "logical replication parallel apply worker for subscription %u", subid);
+	}
+	else if (OidIsValid(relid))
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "TablesyncWorkerMain");
+		snprintf(bgw.bgw_name, BGW_MAXLEN,
+				 "logical replication tablesync worker for subscription %u sync %u", subid, relid);
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication tablesync worker");
+	}
 	else
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
 				 "logical replication apply worker for subscription %u", subid);
-
-	if (is_parallel_apply_worker)
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication parallel worker");
-	else
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker");
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication apply worker");
+	}
 
 	bgw.bgw_restart_time = BGW_NEVER_RESTART;
 	bgw.bgw_notify_pid = MyProcPid;
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 6d461654ab..8125bbd170 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -131,7 +131,7 @@ static StringInfo copybuf = NULL;
 /*
  * Exit routine for synchronization worker.
  */
-static void
+void
 pg_attribute_noreturn()
 finish_sync_worker(void)
 {
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 0ee764d68f..b979a755ae 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -392,6 +392,7 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void set_stream_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -4330,6 +4331,69 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* set_stream_options
+  * 	Set logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name
+  * and origin start position. Workers need these options for logical replication.
+  */
+static void
+set_stream_options(WalRcvStreamOptions *options, char *slotname, XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+		options->proto.logical.twophase = true;
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4442,13 +4506,134 @@ start_apply(XLogRecPtr origin_startpos)
 }
 
 /*
- * Common initialization for leader apply worker and parallel apply worker.
+ * Runs the tablesync worker.
+ * It starts syncing tables. After a successful sync,
+ * sets streaming options and starts streaming to catchup.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Start table synchronization. */
+	start_table_sync(origin_startpos, &slotname);
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	/* Start applying changes to catchup. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Runs the leader apply worker.
+ * It sets up replication origin, streaming options
+ * and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+	bool		must_use_password;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard
+	 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+	 * crash if slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+
+	/* Is the use of a password mandatory? */
+	must_use_password = MySubscription->passwordrequired &&
+		!superuser_arg(MySubscription->owner);
+
+	/* Note that the superuser_arg call can access the DB */
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											must_use_password,
+											MySubscription->name, &err);
+
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it
+	 * does some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+
+	/* Run the main loop. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Common initialization for logical replication workers; leader apply worker,
+ * parallel apply worker and tablesync worker.
  *
  * Initialize the database connection, in-memory subscription and necessary
  * config options.
  */
 void
-InitializeApplyWorker(void)
+InitializeLogRepWorker(void)
 {
 	MemoryContext oldctx;
 
@@ -4512,7 +4697,8 @@ InitializeApplyWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+						get_worker_name(),
 						MySubscription->name,
 						get_rel_name(MyLogicalRepWorker->relid))));
 	else
@@ -4533,7 +4719,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	InitializingApplyWorker = true;
 
@@ -4557,7 +4742,7 @@ ApplyWorkerMain(Datum main_arg)
 	/* Load the libpq-specific functions */
 	load_file("libpqwalreceiver", false);
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
@@ -4565,165 +4750,69 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-		bool		must_use_password;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-
-		/* Is the use of a password mandatory? */
-		must_use_password = MySubscription->passwordrequired &&
-			!superuser_arg(MySubscription->owner);
-
-		/* Note that the superuser_arg call can access the DB */
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												must_use_password,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
-	 * the subscription relation state.
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
 	 */
 	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
+	/* This is leader apply worker */
+	run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
+	proc_exit(0);
+}
 
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
+/* Logical Replication Tablesync worker entry point */
+void
+TablesyncWorkerMain(Datum main_arg)
+{
+	int			worker_slot = DatumGetInt32(main_arg);
+	char		originname[NAMEDATALEN];
+	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
+	char	   *myslotname = NULL;
+	WalRcvStreamOptions options;
+
+	/* Attach to slot */
+	logicalrep_worker_attach(worker_slot);
+
+	/* Setup signal handling */
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
 
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * We don't currently need any ResourceOwner in a walreceiver process, but
+	 * if we did, we could call CreateAuxProcessResourceOwner here.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
 
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
+	/* Initialise stats to a sanish value */
+	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
+		MyLogicalRepWorker->reply_time = GetCurrentTimestamp();
 
-	if (!am_tablesync_worker())
-	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
-		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+	/* Load the libpq-specific functions */
+	load_file("libpqwalreceiver", false);
 
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
-		}
-		else
-		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-		}
+	InitializeLogRepWorker();
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-								 MySubscription->name,
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-								 "?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
+	/* Connect to the origin and start the replication. */
+	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
+		 MySubscription->conninfo);
 
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+	/*
+	 * Setup callback for syscache so that we know when something changes in
+	 * the subscription relation state. Do this outside the loop to avoid
+	 * exceeding MAX_SYSCACHE_CALLBACKS
+	 */
+	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
+								  invalidate_syncing_table_states,
+								  (Datum) 0);
 
-	proc_exit(0);
+	run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+
+	finish_sync_worker();
 }
 
 /*
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index 39588da79f..bbd71d0b42 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -18,6 +18,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending;
 
 extern void ApplyWorkerMain(Datum main_arg);
 extern void ParallelApplyWorkerMain(Datum main_arg);
+extern void TablesyncWorkerMain(Datum main_arg);
 
 extern bool IsLogicalWorker(void);
 extern bool IsLogicalParallelApplyWorker(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 343e781896..7aba034774 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -265,7 +265,7 @@ extern void maybe_reread_subscription(void);
 
 extern void stream_cleanup_files(Oid subid, TransactionId xid);
 
-extern void InitializeApplyWorker(void);
+extern void InitializeLogRepWorker(void);
 
 extern void store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn);
 
@@ -307,6 +307,8 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
+extern void finish_sync_worker(void);
+
 static inline bool
 am_tablesync_worker(void)
 {
-- 
2.27.0

From 8b416d01acd50ab0a7d97d64f122e7ca9903a2b7 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Mon, 5 Jun 2023 15:45:29 +0300
Subject: [PATCH v2 2/6] Reuse Tablesync Workers

This commit allows reusing tablesync workers for syncing more than one
table sequentially during their lifetime, instead of exiting after
only syncing one table.

Before this commit, tablesync workers were capable of syncing only one
table. For each table, a new sync worker was launched and that worker would
exit when done processing the table.

Now, tablesync workers are not limited to processing only one
table. When done, they can move to processing another table in
the same subscription.

If there is a table that needs to be synced, an available tablesync
worker picks up that table and syncs it. Each tablesync worker
continues to pick new tables to sync until there are no tables left
requiring synchronization. If there was no available worker to
process the table, then a new tablesync worker will be launched,
provided the number of tablesync workers for the subscription does not
exceed max_sync_workers_per_subscription.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/replication/logical/launcher.c  |   1 +
 src/backend/replication/logical/tablesync.c |  46 ++++++--
 src/backend/replication/logical/worker.c    | 110 +++++++++++++++++++-
 src/include/replication/worker_internal.h   |   7 ++
 4 files changed, 149 insertions(+), 15 deletions(-)

diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index c49a7fe811..4c3f72cb46 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -440,6 +440,7 @@ retry:
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
+	worker->is_sync_completed = false;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
 	TIMESTAMP_NOBEGIN(worker->last_recv_time);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8125bbd170..37f073b968 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -129,11 +129,10 @@ static bool FetchTableStates(bool *started_tx);
 static StringInfo copybuf = NULL;
 
 /*
- * Exit routine for synchronization worker.
+ * Prepares the synchronization worker for reuse or exit.
  */
 void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
 	 * Commit any outstanding transaction. This is the usual case, unless
@@ -145,19 +144,38 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	if (LogRepWorkerWalRcvConn != NULL)
+	{
+		walrcv_disconnect(LogRepWorkerWalRcvConn);
+		LogRepWorkerWalRcvConn = NULL;
+	}
+
+	/* Find the leader apply worker and signal it. */
+	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
 	/* And flush all writes. */
 	XLogFlush(GetXLogWriteRecPtr());
 
 	StartTransactionCommand();
 	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
+			(errmsg("logical replication table synchronization worker for subscription \"%s\" has finished",
+					MySubscription->name)));
 	CommitTransactionCommand();
 
-	/* Find the leader apply worker and signal it. */
-	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
-
 	/* Stop gracefully */
 	proc_exit(0);
 }
@@ -379,7 +397,15 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 */
 		replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+		/* Sync worker has completed synchronization of the current table. */
+		MyLogicalRepWorker->is_sync_completed = true;
+
+		ereport(LOG,
+		(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has finished",
+				MySubscription->name,
+				get_rel_name(MyLogicalRepWorker->relid),
+				MyLogicalRepWorker->relid)));
+		CommitTransactionCommand();
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index b979a755ae..4a0f402ad4 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3624,6 +3624,20 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can call process_syncing_tables_for_sync.
+				 *
+				 * process_syncing_tables_for_sync decides whether the sync of the
+				 * current table is completed. If it is completed, streaming must
+				 * be already ended. So, we can break the loop.
+				 */
+				if (MyLogicalRepWorker->is_sync_completed)
+				{
+					endofstream = true;
+					break;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3643,6 +3657,15 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+
+			/*
+			 * If is_sync_completed is true, this means that the tablesync worker
+			 * is done with synchronization. Streaming has already been ended by
+			 * process_syncing_tables_for_sync. We should move to the next table
+			 * if needed, or exit.
+			 */
+			if (MyLogicalRepWorker->is_sync_completed)
+				endofstream = true;
 		}
 
 		/* Cleanup the memory. */
@@ -3745,8 +3768,11 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/* Tablesync workers should end streaming before exiting the main loop
+	 * to drop replication slot. Only end streaming here for apply workers.
+	 */
+	if (!am_tablesync_worker())
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
 }
 
 /*
@@ -4517,6 +4543,8 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 					 int originname_size,
 					 XLogRecPtr *origin_startpos)
 {
+	MyLogicalRepWorker->is_sync_completed = false;
+
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname);
 
@@ -4697,10 +4725,11 @@ InitializeLogRepWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" with relid %u has started",
 						get_worker_name(),
 						MySubscription->name,
-						get_rel_name(MyLogicalRepWorker->relid))));
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
 	else
 		ereport(LOG,
 		/* translator: first %s is the name of logical replication worker */
@@ -4810,7 +4839,78 @@ TablesyncWorkerMain(Datum main_arg)
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+	/*
+	 * The loop where worker does its job. It loops until there is no relation
+	 * left to sync.
+	 */
+	for (;;)
+	{
+		List	   *rstates;
+		SubscriptionRelState *rstate;
+		ListCell   *lc;
+		bool 	is_table_found = false;
+
+		run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+
+		if (IsTransactionState())
+			CommitTransactionCommand();
+
+		if (MyLogicalRepWorker->is_sync_completed)
+		{
+			/* This transaction will be committed by clean_sync_worker. */
+			StartTransactionCommand();
+
+			/*
+			 * Check if there is any table whose relation state is still INIT.
+			 * If a table in INIT state is found, the worker will not be finished,
+			 * it will be reused instead.
+			 */
+			rstates = GetSubscriptionRelations(MySubscription->oid, true);
+			rstate = (SubscriptionRelState *) palloc(sizeof(SubscriptionRelState));
+
+			foreach(lc, rstates)
+			{
+				memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
+
+				/*
+				* Pick the table for the next run if it is not already picked up
+				* by another worker.
+				*
+				* Take exclusive lock to prevent any other sync worker from picking
+				* the same table.
+				*/
+				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
+				if (rstate->state != SUBREL_STATE_SYNCDONE &&
+					!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+				{
+					/* Update worker state for the next table */
+					MyLogicalRepWorker->relid = rstate->relid;
+					MyLogicalRepWorker->relstate = rstate->state;
+					MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+					LWLockRelease(LogicalRepWorkerLock);
+
+					/* found a table for next iteration */
+					is_table_found = true;
+					clean_sync_worker();
+
+					StartTransactionCommand();
+					ereport(LOG,
+							(errmsg("%s for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+									get_worker_name(),
+									MySubscription->name,
+									get_rel_name(MyLogicalRepWorker->relid),
+									MyLogicalRepWorker->relid)));
+					CommitTransactionCommand();
+
+					break;
+				}
+				LWLockRelease(LogicalRepWorkerLock);
+			}
+
+			if (!is_table_found)
+				break;
+		}
+	}
 
 	finish_sync_worker();
 }
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 7aba034774..1e9f8e6e72 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -56,6 +56,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Indicates whether tablesync worker has completed sycning its assigned
+	 * table. If true, no need to continue with that table.
+	 */
+	bool		is_sync_completed;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -308,6 +314,7 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
 extern void finish_sync_worker(void);
+extern void clean_sync_worker(void);
 
 static inline bool
 am_tablesync_worker(void)
-- 
2.27.0

From 7e9731104d17056509df919b6b6122e6a3f1fce8 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <kuroda.hayato@fujitsu.com>
Date: Tue, 27 Jun 2023 07:10:45 +0000
Subject: [PATCH v2 3/6] reuse connection when tablesync workers change the
 target

---
 src/backend/replication/logical/tablesync.c | 33 ++++++++++++++++-----
 src/backend/replication/logical/worker.c    | 33 +++++++++++++--------
 src/backend/replication/walsender.c         |  6 ++++
 src/include/replication/worker_internal.h   |  2 +-
 4 files changed, 52 insertions(+), 22 deletions(-)

diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 37f073b968..8d44ed207f 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -1268,7 +1268,7 @@ ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
  * The returned slot name is palloc'ed in current memory context.
  */
 char *
-LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
+LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 {
 	char	   *slotname;
 	char	   *err;
@@ -1321,14 +1321,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 									NAMEDATALEN);
 
 	/*
-	 * Here we use the slot name instead of the subscription name as the
-	 * application_name, so that it is different from the leader apply worker,
-	 * so that synchronous replication can distinguish them.
+	 * Connect to publisher if not yet. The application_name must be also
+	 * different from the leader apply worker because synchronous replication
+	 * must distinguish them.
 	 */
-	LogRepWorkerWalRcvConn =
-		walrcv_connect(MySubscription->conninfo, true,
-					   must_use_password,
-					   slotname, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+	{
+		char application_name[NAMEDATALEN];
+
+		/*
+		 * FIXME: set appropriate application_name. Previously, the slot name
+		 * was used becasue the lifetime of the tablesync worker was same as
+		 * that, but now the tablesync worker handles many slots during the
+		 * synchronization so that it is not suitable. So what should be?
+		 * Note that if the tablesync worker starts to reuse the replication
+		 * slot during synchronization, we should use the slot name as
+		 * application_name again.
+		 */
+		snprintf(application_name, NAMEDATALEN, "pg_%u_sync_%i",
+				 MySubscription->oid, worker_slot);
+		LogRepWorkerWalRcvConn =
+			walrcv_connect(MySubscription->conninfo, true,
+						   must_use_password,
+						   application_name, &err);
+	}
+
 	if (LogRepWorkerWalRcvConn == NULL)
 		ereport(ERROR,
 				(errcode(ERRCODE_CONNECTION_FAILURE),
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 4a0f402ad4..5420675ce8 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3498,19 +3498,21 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 	/*
 	 * Init the ApplyMessageContext which we clean up after each replication
-	 * protocol message.
+	 * protocol message, if needed.
 	 */
-	ApplyMessageContext = AllocSetContextCreate(ApplyContext,
-												"ApplyMessageContext",
-												ALLOCSET_DEFAULT_SIZES);
+	if (!ApplyMessageContext)
+		ApplyMessageContext = AllocSetContextCreate(ApplyContext,
+													"ApplyMessageContext",
+													ALLOCSET_DEFAULT_SIZES);
 
 	/*
 	 * This memory context is used for per-stream data when the streaming mode
 	 * is enabled. This context is reset on each stream stop.
 	 */
-	LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
-													"LogicalStreamingContext",
-													ALLOCSET_DEFAULT_SIZES);
+	if (!LogicalStreamingContext)
+		LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
+														"LogicalStreamingContext",
+														ALLOCSET_DEFAULT_SIZES);
 
 	/* mark as idle, before starting to loop */
 	pgstat_report_activity(STATE_IDLE, NULL);
@@ -4462,7 +4464,7 @@ TwoPhaseTransactionGid(Oid subid, TransactionId xid, char *gid, int szgid)
  * are not repeatable.
  */
 static void
-start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
+start_table_sync(XLogRecPtr *origin_startpos, char **myslotname, int worker_slot)
 {
 	char	   *syncslotname = NULL;
 
@@ -4471,7 +4473,7 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 	PG_TRY();
 	{
 		/* Call initial sync. */
-		syncslotname = LogicalRepSyncTableStart(origin_startpos);
+		syncslotname = LogicalRepSyncTableStart(origin_startpos, worker_slot);
 	}
 	PG_CATCH();
 	{
@@ -4541,12 +4543,13 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 					 char *slotname,
 					 char *originname,
 					 int originname_size,
-					 XLogRecPtr *origin_startpos)
+					 XLogRecPtr *origin_startpos,
+					 int worker_slot)
 {
 	MyLogicalRepWorker->is_sync_completed = false;
 
 	/* Start table synchronization. */
-	start_table_sync(origin_startpos, &slotname);
+	start_table_sync(origin_startpos, &slotname, worker_slot);
 
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
@@ -4850,7 +4853,9 @@ TablesyncWorkerMain(Datum main_arg)
 		ListCell   *lc;
 		bool 	is_table_found = false;
 
-		run_tablesync_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
+		run_tablesync_worker(&options, myslotname, originname,
+							 sizeof(originname), &origin_startpos,
+							 worker_slot);
 
 		if (IsTransactionState())
 			CommitTransactionCommand();
@@ -4891,7 +4896,9 @@ TablesyncWorkerMain(Datum main_arg)
 
 					/* found a table for next iteration */
 					is_table_found = true;
-					clean_sync_worker();
+
+					CommitTransactionCommand();
+					pgstat_report_stat(true);
 
 					StartTransactionCommand();
 					ereport(LOG,
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index d3a136b6f5..429d00f2f0 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1828,6 +1828,12 @@ exec_replication_command(const char *cmd_string)
 				set_ps_display(cmdtag);
 				PreventInTransactionBlock(true, cmdtag);
 
+				/*
+				 * Initialize the flag again because this streaming may be
+				 * second time.
+				 */
+				streamingDoneSending = streamingDoneReceiving = false;
+
 				if (cmd->kind == REPLICATION_KIND_PHYSICAL)
 					StartReplication(cmd);
 				else
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 1e9f8e6e72..d3cabf4033 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -249,7 +249,7 @@ extern int	logicalrep_sync_worker_count(Oid subid);
 
 extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 											   char *originname, Size szoriginname);
-extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos);
+extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot);
 
 extern bool AllTablesyncsReady(void);
 extern void UpdateTwoPhaseState(Oid suboid, char new_state);
-- 
2.27.0

From 28f35dd8310a62c257d843bf69d1672717b80364 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH v2 4/6] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy
a relation and then apply changes by consuming from replication slot.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 ++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 104 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 289 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index b11d9a6ba3..f7d5acee93 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2595,6 +2595,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index dc9c5c82d9..cf88a3b7d4 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -82,6 +82,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -98,7 +100,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -982,6 +985,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * TODO
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("could not start WAL streaming: %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 41243d0187..3c11f8e1dd 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -476,6 +476,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, must obtain a snapshot able to read all tables;
+ *  	if false, one that can read only catalogs is acceptable.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -494,6 +498,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -502,6 +507,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -571,8 +577,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 55a24c02c9..85c5cdb633 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 6035cf4816..c5b2d5b61f 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -486,6 +486,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 429d00f2f0..974ab5ca35 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1284,7 +1285,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1336,6 +1337,98 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	CheckLogicalDecodingRequirements();
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consistent WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1870,6 +1963,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 281626fa6f..f01e5f23a7 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -386,6 +386,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -403,6 +413,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -437,6 +448,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index e941fb6c82..cc023b10fa 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2348,6 +2348,7 @@ ReplicationSlotInvalidationCause
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3855,6 +3856,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.27.0

From 71307d7181a81adc871e892551c0eb8bf9ea5d3f Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH v2 5/6] Reuse Replication Slot and Origin in Tablesync

This commit allows reusing replication slots and origins during tablesync.

Earlier, a tablesync worker was creating a new replication slot and origin
each time it syncs a new table. With this patch, replication
slots/origins can be reusable for tablesync.

This reduces the overhead of creating/dropping replication slots and origins
and improves tablesync speed significantly especially for empty or small tables.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create those first. Otherwise the worker reuses
slot and origin created by the same worker in previous iterations earlier.
Tables in FINISHEDCOPY are expected to have a replication slot and origin.
Slot and origin names for such tables are persisted in
pg_subscription_rel catalog. Tablesync worker can fetch them and proceed
with existing slot and origin of FINISHEDCOPY tables and does not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 247 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 229 +++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   8 +-
 src/backend/replication/logical/tablesync.c   | 369 +++++++++++++-----
 src/backend/replication/logical/worker.c      |  62 ++-
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  14 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  23 +-
 11 files changed, 797 insertions(+), 198 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 852cb30ae1..60718ab587 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8030,6 +8030,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8114,6 +8127,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index d07f88ce28..152fdaa310 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -108,6 +108,14 @@ GetSubscription(Oid subid, bool missing_ok)
 								   Anum_pg_subscription_suborigin);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -199,6 +207,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -228,7 +274,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -257,6 +303,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -269,6 +325,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -299,14 +409,56 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -318,6 +470,85 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 54895ba929..9f5a295ddc 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -710,6 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -773,7 +774,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -864,6 +865,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
 	bool		must_use_password;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -943,7 +946,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -967,6 +970,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -993,13 +997,36 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
 				 * tablesync origin.
 				 */
-				if (state != SUBREL_STATE_READY)
+				if (state != SUBREL_STATE_READY &&
+					state != SUBREL_STATE_SYNCDONE)
 				{
 					char		originname[NAMEDATALEN];
 
@@ -1027,31 +1054,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1474,6 +1494,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1546,6 +1567,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1595,6 +1624,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1606,36 +1637,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1688,39 +1709,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1743,6 +1742,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2155,6 +2219,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2165,18 +2230,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 1d4e83c4c1..d2c70dffbc 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										InvalidRepSlotId);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 4c3f72cb46..6d49e46dbb 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -1180,7 +1183,8 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID,
+										 InvalidRepSlotId);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8d44ed207f..e2b535f920 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -328,40 +328,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication tracking origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -370,32 +359,70 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
-		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/*
+			 * Drop replication origin
+			 *
+			 * There is a chance that the user is concurrently performing refresh
+			 * for the subscription where we remove the table state and its origin
+			 * or the apply worker would have removed this origin. So passing
+			 * missing_ok = true.
+			 */
+			replorigin_drop_by_name(originname, true, false);
+		}
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+		ereport(DEBUG2,
+			(errmsg("process_syncing_tables_for_sync: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					"NULL", "NULL", MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
 
 		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		StartTransactionCommand();
+		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+										   MyLogicalRepWorker->relid,
+										   originname,
+										   sizeof(originname));
 
 		/* Sync worker has completed synchronization of the current table. */
 		MyLogicalRepWorker->is_sync_completed = true;
@@ -492,6 +519,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -512,18 +540,31 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+				ereport(DEBUG2,
+					(errmsg("process_syncing_tables_for_apply: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+							"NULL", "NULL", rstate->state,
+							rstate->relid, MyLogicalRepWorker->subid)));
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -612,12 +653,25 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+						ereport(DEBUG2,
+								(errmsg("process_syncing_tables_for_apply: incremented lastusedid to %lld for subscription %u",
+										(long long) MySubscription->lastusedid, MySubscription->oid)));
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1240,8 +1294,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1252,11 +1306,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1282,6 +1336,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 	UserContext ucxt;
 	bool		must_use_password;
 	bool		run_as_owner;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1316,7 +1371,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1355,12 +1410,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1374,10 +1443,53 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+			ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL", "NULL", MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		memcpy(slotname, prev_slotname, NAMEDATALEN);
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1397,7 +1509,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1405,11 +1519,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+
+	/*
+	 * Refresh the originname in case of having non-existing origin
+	 * from previous failed sync attempts.
+	 * If that's the case, it should be removed from the catalog so far.
+	 * Then, we can continue by reusing the origin created by the current
+	 * worker instead of .
+	 */
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									MyLogicalRepWorker->relid,
+									originname,
+									sizeof(originname));
+
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					slotname, originname, MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
 	pgstat_report_stat(true);
 
 	StartTransactionCommand();
@@ -1437,48 +1571,95 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: created replication slot %s for subscription %u",
+						slotname, MyLogicalRepWorker->subid)));
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: reusing replication slot %s for relation %u in subscription %u",
+						slotname, MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/*
 	 * Make sure that the copy command runs as the table owner, unless the
 	 * user has opted out of that behaviour.
@@ -1537,12 +1718,18 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					originname, slotname, SUBREL_STATE_FINISHEDCOPY,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 
 copy_table_done:
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 5420675ce8..f7a8db45ee 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -464,8 +464,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -3770,7 +3778,8 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* Tablesync workers should end streaming before exiting the main loop
+	/*
+	 * Tablesync workers should end streaming before exiting the main loop
 	 * to drop replication slot. Only end streaming here for apply workers.
 	 */
 	if (!am_tablesync_worker())
@@ -4496,6 +4505,9 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname, int worker_slot
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4551,10 +4563,12 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname, worker_slot);
 
+	StartTransactionCommand();
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   originname_size);
+	CommitTransactionCommand();
 
 	set_apply_error_context_origin(originname);
 
@@ -4595,11 +4609,10 @@ run_apply_worker(WalRcvStreamOptions *options,
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 					errmsg("subscription has no replication slot set")));
 
-	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-									   originname, originname_size);
-
 	/* Setup replication origin tracking. */
 	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
 	originid = replorigin_by_name(originname, true);
 	if (!OidIsValid(originid))
 		originid = replorigin_create(originname);
@@ -4878,12 +4891,12 @@ TablesyncWorkerMain(Datum main_arg)
 				memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState));
 
 				/*
-				* Pick the table for the next run if it is not already picked up
-				* by another worker.
-				*
-				* Take exclusive lock to prevent any other sync worker from picking
-				* the same table.
-				*/
+				 * Pick the table for the next run if it is not already picked up
+				 * by another worker.
+				 *
+				 * Take exclusive lock to prevent any other sync worker from picking
+				 * the same table.
+				 */
 				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
 				if (rstate->state != SUBREL_STATE_SYNCDONE &&
 					!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
@@ -4915,7 +4928,32 @@ TablesyncWorkerMain(Datum main_arg)
 			}
 
 			if (!is_table_found)
+			{
+				TimeLineID	tli;
+
+				/*
+				 * It is important to give an error if we are unable to drop the
+				 * slot, otherwise, it won't be dropped till the corresponding
+				 * subscription is dropped. So passing missing_ok = false.
+				 */
+				if (MyLogicalRepWorker->created_slot)
+				{
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+					ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+				}
+
+				/*
+				 * Drop replication origin before exiting.
+				 *
+				 * There is a chance that the user is concurrently performing refresh
+				 * for the subscription where we remove the table state and its origin
+				 * or the apply worker would have removed this origin. So passing
+				 * missing_ok = true.
+				 */
+				replorigin_drop_by_name(originname, true, false);
+
 				break;
+			}
 		}
 	}
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 1d40eebc78..7e13f59847 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -108,6 +108,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -144,6 +147,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -164,6 +169,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..185164d75e 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,16 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index a8a89dc784..31b2c41893 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -237,8 +237,9 @@ extern bool InvalidateObsoleteReplicationSlots(ReplicationSlotInvalidationCause
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index d3cabf4033..be8aecfc16 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -239,7 +256,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
@@ -335,4 +353,7 @@ am_parallel_apply_worker(void)
 	return isParallelApplyWorker(MyLogicalRepWorker);
 }
 
+/* Invalid identifier to be used for naming replication slots */
+#define InvalidRepSlotId	0
+
 #endif							/* WORKER_INTERNAL_H */
-- 
2.27.0

From f8c891e232da418c8bca614ea5a6825a6ed55e08 Mon Sep 17 00:00:00 2001
From: Hayato Kuroda <kuroda.hayato@fujitsu.com>
Date: Wed, 28 Jun 2023 06:19:34 +0000
Subject: [PATCH v2 6/6] Use slot name as application_name again

---
 src/backend/replication/logical/tablesync.c | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index e2b535f920..9728541acf 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -1382,23 +1382,14 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 	 */
 	if (LogRepWorkerWalRcvConn == NULL)
 	{
-		char application_name[NAMEDATALEN];
-
 		/*
-		 * FIXME: set appropriate application_name. Previously, the slot name
-		 * was used becasue the lifetime of the tablesync worker was same as
-		 * that, but now the tablesync worker handles many slots during the
-		 * synchronization so that it is not suitable. So what should be?
-		 * Note that if the tablesync worker starts to reuse the replication
-		 * slot during synchronization, we should use the slot name as
-		 * application_name again.
+		 * Here we use the slot name instead of the subscription name as the
+		 * application_name,
 		 */
-		snprintf(application_name, NAMEDATALEN, "pg_%u_sync_%i",
-				 MySubscription->oid, worker_slot);
 		LogRepWorkerWalRcvConn =
 			walrcv_connect(MySubscription->conninfo, true,
 						   must_use_password,
-						   application_name, &err);
+						   slotname, &err);
 	}
 
 	if (LogRepWorkerWalRcvConn == NULL)
-- 
2.27.0

From a76e29807608fd3ca837483fd8c6c6caf6c3a407 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Mon, 5 Jun 2023 15:04:41 +0300
Subject: [PATCH 1/5] Refactor to split Apply and Tablesync Workers

Both apply and tablesync workers were using ApplyWorkerMain() as entry
point. As the name implies, ApplyWorkerMain() should be considered as
the main function for apply workers. Tablesync worker's path was hidden
and does not have enough in common to share the same main function with
apply worker.

Also; most of the code shared by both worker types are already combined
in LogicalRepApplyLoop(). There is no need to combine the rest in
ApplyWorkerMain() anymore.

This commit introduces TablesyncWorkerMain() as a new entry point for
tablesync workers and separates both type of workers from each other.
This aims to increase code readability and help to maintain logical
replication workers separately.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/postmaster/bgworker.c             |   3 +
 .../replication/logical/applyparallelworker.c |   2 +-
 src/backend/replication/logical/launcher.c    |  32 +-
 src/backend/replication/logical/tablesync.c   |   2 +-
 src/backend/replication/logical/worker.c      | 388 +++++++++++-------
 src/include/replication/logicalworker.h       |   1 +
 src/include/replication/worker_internal.h     |   4 +-
 7 files changed, 270 insertions(+), 162 deletions(-)

diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 5b4bd71694..505e38376c 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -131,6 +131,9 @@ static const struct
 	},
 	{
 		"ParallelApplyWorkerMain", ParallelApplyWorkerMain
+	},
+	{
+		"TablesyncWorkerMain", TablesyncWorkerMain
 	}
 };
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 6fb96148f4..1d4e83c4c1 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -942,7 +942,7 @@ ParallelApplyWorkerMain(Datum main_arg)
 	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
 		MyLogicalRepWorker->reply_time = 0;
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 542af7d863..e231fa7f95 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -459,24 +459,30 @@ retry:
 	snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres");
 
 	if (is_parallel_apply_worker)
+	{
 		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ParallelApplyWorkerMain");
-	else
-		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
-
-	if (OidIsValid(relid))
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
-				 "logical replication worker for subscription %u sync %u", subid, relid);
-	else if (is_parallel_apply_worker)
+				 "logical replication parallel apply worker for subscription %u",
+				 subid);
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication parallel worker");
+	}
+	else if (OidIsValid(relid))
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "TablesyncWorkerMain");
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
-				 "logical replication parallel apply worker for subscription %u", subid);
+				 "logical replication tablesync worker for subscription %u sync %u",
+				 subid,
+				 relid);
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication tablesync worker");
+	}
 	else
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
-				 "logical replication apply worker for subscription %u", subid);
-
-	if (is_parallel_apply_worker)
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication parallel worker");
-	else
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker");
+				 "logical replication apply worker for subscription %u",
+				 subid);
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication apply worker");
+	}
 
 	bgw.bgw_restart_time = BGW_NEVER_RESTART;
 	bgw.bgw_notify_pid = MyProcPid;
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 6d461654ab..8125bbd170 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -131,7 +131,7 @@ static StringInfo copybuf = NULL;
 /*
  * Exit routine for synchronization worker.
  */
-static void
+void
 pg_attribute_noreturn()
 finish_sync_worker(void)
 {
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 0ee764d68f..96ddad356b 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -392,6 +392,9 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void set_stream_options(WalRcvStreamOptions *options,
+							   char *slotname,
+							   XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -4330,6 +4333,72 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* set_stream_options
+  * 	Set logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name and
+  * origin start position. Workers need these options for logical replication.
+  */
+static void
+set_stream_options(WalRcvStreamOptions *options,
+				   char *slotname,
+				   XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+		options->proto.logical.twophase = true;
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4433,7 +4502,8 @@ start_apply(XLogRecPtr origin_startpos)
 			 * idle state.
 			 */
 			AbortOutOfAnyTransaction();
-			pgstat_report_subscription_error(MySubscription->oid, !am_tablesync_worker());
+			pgstat_report_subscription_error(MySubscription->oid,
+											 !am_tablesync_worker());
 
 			PG_RE_THROW();
 		}
@@ -4442,13 +4512,133 @@ start_apply(XLogRecPtr origin_startpos)
 }
 
 /*
- * Common initialization for leader apply worker and parallel apply worker.
+ * Runs the tablesync worker.
+ * It starts syncing tables. After a successful sync, sets streaming options
+ * and starts streaming to catchup.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Start table synchronization. */
+	start_table_sync(origin_startpos, &slotname);
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	/* Start applying changes to catchup. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Runs the leader apply worker.
+ * It sets up replication origin, streaming options and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+	bool		must_use_password;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard
+	 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+	 * crash if slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+
+	/* Is the use of a password mandatory? */
+	must_use_password = MySubscription->passwordrequired &&
+		!superuser_arg(MySubscription->owner);
+
+	/* Note that the superuser_arg call can access the DB */
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											must_use_password,
+											MySubscription->name, &err);
+
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it
+	 * does some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+
+	/* Run the main loop. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Common initialization for logical replication workers; leader apply worker,
+ * parallel apply worker and tablesync worker.
  *
  * Initialize the database connection, in-memory subscription and necessary
  * config options.
  */
 void
-InitializeApplyWorker(void)
+InitializeLogRepWorker(void)
 {
 	MemoryContext oldctx;
 
@@ -4512,7 +4702,8 @@ InitializeApplyWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+						get_worker_name(),
 						MySubscription->name,
 						get_rel_name(MyLogicalRepWorker->relid))));
 	else
@@ -4533,7 +4724,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	InitializingApplyWorker = true;
 
@@ -4557,7 +4747,7 @@ ApplyWorkerMain(Datum main_arg)
 	/* Load the libpq-specific functions */
 	load_file("libpqwalreceiver", false);
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
@@ -4565,71 +4755,6 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-		bool		must_use_password;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-
-		/* Is the use of a password mandatory? */
-		must_use_password = MySubscription->passwordrequired &&
-			!superuser_arg(MySubscription->owner);
-
-		/* Note that the superuser_arg call can access the DB */
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												must_use_password,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
 	 * the subscription relation state.
@@ -4638,92 +4763,63 @@ ApplyWorkerMain(Datum main_arg)
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
+	/* This is leader apply worker */
+	run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
+	proc_exit(0);
+}
 
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
+/* Logical Replication Tablesync worker entry point */
+void
+TablesyncWorkerMain(Datum main_arg)
+{
+	int			worker_slot = DatumGetInt32(main_arg);
+	char		originname[NAMEDATALEN];
+	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
+	char	   *myslotname = NULL;
+	WalRcvStreamOptions options;
+
+	/* Attach to slot */
+	logicalrep_worker_attach(worker_slot);
+
+	/* Setup signal handling */
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
 
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * We don't currently need any ResourceOwner in a walreceiver process, but
+	 * if we did, we could call CreateAuxProcessResourceOwner here.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
 
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
+	/* Initialise stats to a sanish value */
+	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
+		MyLogicalRepWorker->reply_time = GetCurrentTimestamp();
 
-	if (!am_tablesync_worker())
-	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
-		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+	/* Load the libpq-specific functions */
+	load_file("libpqwalreceiver", false);
 
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
-		}
-		else
-		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-		}
+	InitializeLogRepWorker();
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-								 MySubscription->name,
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-								 "?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
+	/* Connect to the origin and start the replication. */
+	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
+		 MySubscription->conninfo);
 
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+	/*
+	 * Setup callback for syscache so that we know when something changes in
+	 * the subscription relation state.
+	 */
+	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
+								  invalidate_syncing_table_states,
+								  (Datum) 0);
 
-	proc_exit(0);
+	run_tablesync_worker(&options,
+						 myslotname,
+						 originname,
+						 sizeof(originname),
+						 &origin_startpos);
+
+	finish_sync_worker();
 }
 
 /*
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index 39588da79f..bbd71d0b42 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -18,6 +18,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending;
 
 extern void ApplyWorkerMain(Datum main_arg);
 extern void ParallelApplyWorkerMain(Datum main_arg);
+extern void TablesyncWorkerMain(Datum main_arg);
 
 extern bool IsLogicalWorker(void);
 extern bool IsLogicalParallelApplyWorker(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 343e781896..7aba034774 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -265,7 +265,7 @@ extern void maybe_reread_subscription(void);
 
 extern void stream_cleanup_files(Oid subid, TransactionId xid);
 
-extern void InitializeApplyWorker(void);
+extern void InitializeLogRepWorker(void);
 
 extern void store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn);
 
@@ -307,6 +307,8 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
+extern void finish_sync_worker(void);
+
 static inline bool
 am_tablesync_worker(void)
 {
-- 
2.25.1

From d482022b40e0a5ce1b74fd0e320cb5b45da2f671 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Tue, 4 Jul 2023 22:04:46 +0300
Subject: [PATCH 2/5] Reuse Tablesync Workers

This commit allows reusing tablesync workers for syncing more than one
table sequentially during their lifetime, instead of exiting after
only syncing one table.

Before this commit, tablesync workers were capable of syncing only one
table. For each table, a new sync worker was launched and that worker would
exit when done processing the table.

Now, tablesync workers are not limited to processing only one
table. When done, they can move to processing another table in
the same subscription.

If there is a table that needs to be synced, an available tablesync
worker picks up that table and syncs it. Each tablesync worker
continues to pick new tables to sync until there are no tables left
requiring synchronization. If there was no available worker to
process the table, then a new tablesync worker will be launched,
provided the number of tablesync workers for the subscription does not
exceed max_sync_workers_per_subscription.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/replication/logical/launcher.c  |   1 +
 src/backend/replication/logical/tablesync.c |  50 ++++++--
 src/backend/replication/logical/worker.c    | 121 ++++++++++++++++++--
 src/include/replication/worker_internal.h   |   7 ++
 4 files changed, 158 insertions(+), 21 deletions(-)

diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index e231fa7f95..72e5ef8a78 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -440,6 +440,7 @@ retry:
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
+	worker->is_sync_completed = false;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
 	TIMESTAMP_NOBEGIN(worker->last_recv_time);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8125bbd170..605c5bd4ec 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -129,15 +129,14 @@ static bool FetchTableStates(bool *started_tx);
 static StringInfo copybuf = NULL;
 
 /*
- * Exit routine for synchronization worker.
+ * Prepares the synchronization worker for reuse or exit.
  */
 void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
-	 * Commit any outstanding transaction. This is the usual case, unless
-	 * there was nothing to do for the table.
+	 * Commit any outstanding transaction. This is the usual case, unless there
+	 * was nothing to do for the table.
 	 */
 	if (IsTransactionState())
 	{
@@ -145,19 +144,38 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	if (LogRepWorkerWalRcvConn != NULL)
+	{
+		walrcv_disconnect(LogRepWorkerWalRcvConn);
+		LogRepWorkerWalRcvConn = NULL;
+	}
+
+	/* Find the leader apply worker and signal it. */
+	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
 	/* And flush all writes. */
 	XLogFlush(GetXLogWriteRecPtr());
 
 	StartTransactionCommand();
 	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
+			(errmsg("logical replication table synchronization worker for subscription \"%s\" has finished",
+					MySubscription->name)));
 	CommitTransactionCommand();
 
-	/* Find the leader apply worker and signal it. */
-	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
-
 	/* Stop gracefully */
 	proc_exit(0);
 }
@@ -379,7 +397,15 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 */
 		replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+		/* Sync worker has completed synchronization of the current table. */
+		MyLogicalRepWorker->is_sync_completed = true;
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
+		CommitTransactionCommand();
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 96ddad356b..eae561db05 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3626,6 +3626,20 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can call process_syncing_tables_for_sync.
+				 *
+				 * process_syncing_tables_for_sync decides whether the sync of
+				 * the current table is completed. If it is completed,
+				 * streaming must be already ended. So, we can break the loop.
+				 */
+				if (MyLogicalRepWorker->is_sync_completed)
+				{
+					endofstream = true;
+					break;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3645,6 +3659,15 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+
+			/*
+			 * If is_sync_completed is true, this means that the tablesync
+			 * worker is done with synchronization. Streaming has already been
+			 * ended by process_syncing_tables_for_sync. We should move to the
+			 * next table if needed, or exit.
+			 */
+			if (MyLogicalRepWorker->is_sync_completed)
+				endofstream = true;
 		}
 
 		/* Cleanup the memory. */
@@ -3747,8 +3770,12 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * Tablesync workers should end streaming before exiting the main loop to
+	 * drop replication slot. Only end streaming here for apply workers.
+	 */
+	if (!am_tablesync_worker())
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
 }
 
 /*
@@ -4523,6 +4550,8 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 					 int originname_size,
 					 XLogRecPtr *origin_startpos)
 {
+	MyLogicalRepWorker->is_sync_completed = false;
+
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname);
 
@@ -4702,10 +4731,11 @@ InitializeLogRepWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" with relid %u has started",
 						get_worker_name(),
 						MySubscription->name,
-						get_rel_name(MyLogicalRepWorker->relid))));
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
 	else
 		ereport(LOG,
 		/* translator: first %s is the name of logical replication worker */
@@ -4813,11 +4843,84 @@ TablesyncWorkerMain(Datum main_arg)
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	run_tablesync_worker(&options,
-						 myslotname,
-						 originname,
-						 sizeof(originname),
-						 &origin_startpos);
+	/*
+	 * The loop where worker does its job. It loops until there is no relation
+	 * left to sync.
+	 */
+	for (;;)
+	{
+		List	   *rstates;
+		ListCell   *lc;
+		bool 	is_table_found = false;
+
+		run_tablesync_worker(&options,
+							 myslotname,
+							 originname,
+							 sizeof(originname),
+							 &origin_startpos);
+
+		if (IsTransactionState())
+			CommitTransactionCommand();
+
+		if (MyLogicalRepWorker->is_sync_completed)
+		{
+			/* This transaction will be committed by clean_sync_worker. */
+			StartTransactionCommand();
+
+			/*
+			 * Check if there is any table whose relation state is still INIT.
+			 * If a table in INIT state is found, the worker will not be
+			 * finished, it will be reused instead.
+			 */
+			rstates = GetSubscriptionRelations(MySubscription->oid, true);
+
+			foreach(lc, rstates)
+			{
+				SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
+
+				if (rstate->state == SUBREL_STATE_SYNCDONE)
+					continue;
+
+				/*
+				 * Take exclusive lock to prevent any other sync worker from
+				 * picking the same table.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
+
+				/*
+				 * Pick the table for the next run if it is not already picked
+				 * up by another worker.
+				 */
+				if (!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+				{
+					/* Update worker state for the next table */
+					MyLogicalRepWorker->relid = rstate->relid;
+					MyLogicalRepWorker->relstate = rstate->state;
+					MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+					LWLockRelease(LogicalRepWorkerLock);
+
+					/* Found a table for next iteration */
+					is_table_found = true;
+					clean_sync_worker();
+
+					StartTransactionCommand();
+					ereport(LOG,
+							(errmsg("%s for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+									get_worker_name(),
+									MySubscription->name,
+									get_rel_name(MyLogicalRepWorker->relid),
+									MyLogicalRepWorker->relid)));
+					CommitTransactionCommand();
+
+					break;
+				}
+				LWLockRelease(LogicalRepWorkerLock);
+			}
+
+			if (!is_table_found)
+				break;
+		}
+	}
 
 	finish_sync_worker();
 }
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 7aba034774..1e9f8e6e72 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -56,6 +56,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Indicates whether tablesync worker has completed sycning its assigned
+	 * table. If true, no need to continue with that table.
+	 */
+	bool		is_sync_completed;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -308,6 +314,7 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
 extern void finish_sync_worker(void);
+extern void clean_sync_worker(void);
 
 static inline bool
 am_tablesync_worker(void)
-- 
2.25.1

From ae042ea7aabacfa6a97960d4e5e3c8810bffd74e Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Tue, 4 Jul 2023 22:13:52 +0300
Subject: [PATCH 3/5] reuse connection when tablesync workers change the target

---
 src/backend/replication/logical/tablesync.c | 53 ++++++++++++++-------
 src/backend/replication/logical/worker.c    | 40 +++++++++++-----
 src/backend/replication/walsender.c         |  6 +++
 src/include/replication/worker_internal.h   |  3 +-
 4 files changed, 71 insertions(+), 31 deletions(-)

diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 605c5bd4ec..f042d9ae00 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -144,16 +144,6 @@ clean_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/*
-	 * Disconnect from publisher. Otherwise reused sync workers causes
-	 * exceeding max_wal_senders
-	 */
-	if (LogRepWorkerWalRcvConn != NULL)
-	{
-		walrcv_disconnect(LogRepWorkerWalRcvConn);
-		LogRepWorkerWalRcvConn = NULL;
-	}
-
 	/* Find the leader apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
 }
@@ -167,6 +157,16 @@ finish_sync_worker(void)
 {
 	clean_sync_worker();
 
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders.
+	 */
+	if (LogRepWorkerWalRcvConn != NULL)
+	{
+		walrcv_disconnect(LogRepWorkerWalRcvConn);
+		LogRepWorkerWalRcvConn = NULL;
+	}
+
 	/* And flush all writes. */
 	XLogFlush(GetXLogWriteRecPtr());
 
@@ -1268,7 +1268,7 @@ ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
  * The returned slot name is palloc'ed in current memory context.
  */
 char *
-LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
+LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 {
 	char	   *slotname;
 	char	   *err;
@@ -1321,14 +1321,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 									NAMEDATALEN);
 
 	/*
-	 * Here we use the slot name instead of the subscription name as the
-	 * application_name, so that it is different from the leader apply worker,
-	 * so that synchronous replication can distinguish them.
+	 * Connect to publisher if not yet. The application_name must be also
+	 * different from the leader apply worker because synchronous replication
+	 * must distinguish them.
 	 */
-	LogRepWorkerWalRcvConn =
-		walrcv_connect(MySubscription->conninfo, true,
-					   must_use_password,
-					   slotname, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+	{
+		char application_name[NAMEDATALEN];
+
+		/*
+		 * FIXME: set appropriate application_name. Previously, the slot name
+		 * was used becasue the lifetime of the tablesync worker was same as
+		 * that, but now the tablesync worker handles many slots during the
+		 * synchronization so that it is not suitable. So what should be?
+		 * Note that if the tablesync worker starts to reuse the replication
+		 * slot during synchronization, we should use the slot name as
+		 * application_name again.
+		 */
+		snprintf(application_name, NAMEDATALEN, "pg_%u_sync_%i",
+				 MySubscription->oid, worker_slot);
+		LogRepWorkerWalRcvConn =
+			walrcv_connect(MySubscription->conninfo, true,
+						   must_use_password,
+						   application_name, &err);
+	}
+
 	if (LogRepWorkerWalRcvConn == NULL)
 		ereport(ERROR,
 				(errcode(ERRCODE_CONNECTION_FAILURE),
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index eae561db05..537cd33a30 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3500,19 +3500,21 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 	/*
 	 * Init the ApplyMessageContext which we clean up after each replication
-	 * protocol message.
+	 * protocol message, if needed.
 	 */
-	ApplyMessageContext = AllocSetContextCreate(ApplyContext,
-												"ApplyMessageContext",
-												ALLOCSET_DEFAULT_SIZES);
+	if (!ApplyMessageContext)
+		ApplyMessageContext = AllocSetContextCreate(ApplyContext,
+													"ApplyMessageContext",
+													ALLOCSET_DEFAULT_SIZES);
 
 	/*
 	 * This memory context is used for per-stream data when the streaming mode
 	 * is enabled. This context is reset on each stream stop.
 	 */
-	LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
-													"LogicalStreamingContext",
-													ALLOCSET_DEFAULT_SIZES);
+	if (!LogicalStreamingContext)
+		LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
+														"LogicalStreamingContext",
+														ALLOCSET_DEFAULT_SIZES);
 
 	/* mark as idle, before starting to loop */
 	pgstat_report_activity(STATE_IDLE, NULL);
@@ -4468,7 +4470,9 @@ TwoPhaseTransactionGid(Oid subid, TransactionId xid, char *gid, int szgid)
  * are not repeatable.
  */
 static void
-start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
+start_table_sync(XLogRecPtr *origin_startpos,
+				 char **myslotname,
+				 int worker_slot)
 {
 	char	   *syncslotname = NULL;
 
@@ -4477,7 +4481,7 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 	PG_TRY();
 	{
 		/* Call initial sync. */
-		syncslotname = LogicalRepSyncTableStart(origin_startpos);
+		syncslotname = LogicalRepSyncTableStart(origin_startpos, worker_slot);
 	}
 	PG_CATCH();
 	{
@@ -4548,12 +4552,23 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 					 char *slotname,
 					 char *originname,
 					 int originname_size,
-					 XLogRecPtr *origin_startpos)
+					 XLogRecPtr *origin_startpos,
+					 int worker_slot)
 {
 	MyLogicalRepWorker->is_sync_completed = false;
 
+	/*
+	 * If it's already connected to the publisher, end streaming before using
+	 * the same connection for another iteration
+	 */
+	if (LogRepWorkerWalRcvConn != NULL)
+	{
+		TimeLineID tli;
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
+
 	/* Start table synchronization. */
-	start_table_sync(origin_startpos, &slotname);
+	start_table_sync(origin_startpos, &slotname, worker_slot);
 
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
@@ -4857,7 +4872,8 @@ TablesyncWorkerMain(Datum main_arg)
 							 myslotname,
 							 originname,
 							 sizeof(originname),
-							 &origin_startpos);
+							 &origin_startpos,
+							 worker_slot);
 
 		if (IsTransactionState())
 			CommitTransactionCommand();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index d3a136b6f5..429d00f2f0 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1828,6 +1828,12 @@ exec_replication_command(const char *cmd_string)
 				set_ps_display(cmdtag);
 				PreventInTransactionBlock(true, cmdtag);
 
+				/*
+				 * Initialize the flag again because this streaming may be
+				 * second time.
+				 */
+				streamingDoneSending = streamingDoneReceiving = false;
+
 				if (cmd->kind == REPLICATION_KIND_PHYSICAL)
 					StartReplication(cmd);
 				else
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 1e9f8e6e72..af6fd339f7 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -249,7 +249,8 @@ extern int	logicalrep_sync_worker_count(Oid subid);
 
 extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 											   char *originname, Size szoriginname);
-extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos);
+extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos,
+									  int worker_slot);
 
 extern bool AllTablesyncsReady(void);
 extern void UpdateTwoPhaseState(Oid suboid, char new_state);
-- 
2.25.1

From 55a72eee60c14d18d84c93f4ec0dc5b70648406d Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH 4/5] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy
a relation and then apply changes by consuming from replication slot.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 ++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 104 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 289 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index b11d9a6ba3..f7d5acee93 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2595,6 +2595,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index dc9c5c82d9..17358bf552 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -82,6 +82,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -98,7 +100,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -982,6 +985,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * Create a new snapshot by using an existing replication slot.
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("Could not create a snapshot by replication slot %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("Could not create a snapshot by replication slot %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 41243d0187..3c11f8e1dd 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -476,6 +476,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, must obtain a snapshot able to read all tables;
+ *  	if false, one that can read only catalogs is acceptable.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -494,6 +498,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -502,6 +507,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -571,8 +577,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 55a24c02c9..85c5cdb633 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 6035cf4816..c5b2d5b61f 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -486,6 +486,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 429d00f2f0..974ab5ca35 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1284,7 +1285,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1336,6 +1337,98 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	CheckLogicalDecodingRequirements();
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consistent WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1870,6 +1963,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 281626fa6f..f01e5f23a7 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -386,6 +386,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -403,6 +413,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -437,6 +448,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index e941fb6c82..cc023b10fa 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2348,6 +2348,7 @@ ReplicationSlotInvalidationCause
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3855,6 +3856,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.25.1

From e429474581810c668af7f03d16a17f112a4e295f Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH 5/5] Reuse Replication Slot and Origin in Tablesync

This commit allows reusing replication slots and origins during tablesync.

Earlier, a tablesync worker was creating a new replication slot and origin
each time it syncs a new table. With this patch, replication
slots/origins can be reusable for tablesync.

This reduces the overhead of creating/dropping replication slots and origins
and improves tablesync speed significantly especially for empty or small tables.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create those first. Otherwise the worker reuses
slot and origin created by the same worker in previous iterations earlier.
Tables in FINISHEDCOPY are expected to have a replication slot and origin.
Slot and origin names for such tables are persisted in
pg_subscription_rel catalog. Tablesync worker can fetch them and proceed
with existing slot and origin of FINISHEDCOPY tables and does not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 247 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 229 +++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   8 +-
 src/backend/replication/logical/tablesync.c   | 373 +++++++++++++-----
 src/backend/replication/logical/worker.c      |  47 ++-
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  14 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  23 +-
 11 files changed, 793 insertions(+), 191 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 852cb30ae1..60718ab587 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8030,6 +8030,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8114,6 +8127,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index d07f88ce28..152fdaa310 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -108,6 +108,14 @@ GetSubscription(Oid subid, bool missing_ok)
 								   Anum_pg_subscription_suborigin);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -199,6 +207,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -228,7 +274,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -257,6 +303,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -269,6 +325,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -299,14 +409,56 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -318,6 +470,85 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index 54895ba929..9f5a295ddc 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -710,6 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -773,7 +774,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -864,6 +865,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
 	bool		must_use_password;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -943,7 +946,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -967,6 +970,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -993,13 +997,36 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
 				 * tablesync origin.
 				 */
-				if (state != SUBREL_STATE_READY)
+				if (state != SUBREL_STATE_READY &&
+					state != SUBREL_STATE_SYNCDONE)
 				{
 					char		originname[NAMEDATALEN];
 
@@ -1027,31 +1054,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1474,6 +1494,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1546,6 +1567,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1595,6 +1624,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1606,36 +1637,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1688,39 +1709,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1743,6 +1742,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2155,6 +2219,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2165,18 +2230,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 1d4e83c4c1..d2c70dffbc 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										InvalidRepSlotId);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 72e5ef8a78..c84193319c 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -1183,7 +1186,8 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID,
+										 InvalidRepSlotId);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index f042d9ae00..830fa4696c 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -328,40 +328,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication tracking origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -370,32 +359,73 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
-		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/*
+			 * Drop replication origin
+			 *
+			 * There is a chance that the user is concurrently performing refresh
+			 * for the subscription where we remove the table state and its origin
+			 * or the apply worker would have removed this origin. So passing
+			 * missing_ok = true.
+			 */
+			replorigin_drop_by_name(originname, true, false);
+		}
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+		ereport(DEBUG2,
+				(errmsg("process_syncing_tables_for_sync: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL",
+						"NULL",
+						MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid,
+						MyLogicalRepWorker->subid)));
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
 
 		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		StartTransactionCommand();
+		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+										   MyLogicalRepWorker->relid,
+										   originname,
+										   sizeof(originname));
 
 		/* Sync worker has completed synchronization of the current table. */
 		MyLogicalRepWorker->is_sync_completed = true;
@@ -492,6 +522,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -512,18 +543,31 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+				ereport(DEBUG2,
+					(errmsg("process_syncing_tables_for_apply: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+							"NULL", "NULL", rstate->state,
+							rstate->relid, MyLogicalRepWorker->subid)));
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -612,12 +656,25 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+						ereport(DEBUG2,
+								(errmsg("process_syncing_tables_for_apply: incremented lastusedid to %lld for subscription %u",
+										(long long) MySubscription->lastusedid, MySubscription->oid)));
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1240,8 +1297,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1252,11 +1309,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1282,6 +1339,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 	UserContext ucxt;
 	bool		must_use_password;
 	bool		run_as_owner;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1316,7 +1374,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1355,12 +1413,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1374,10 +1446,53 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+			ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL", "NULL", MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		memcpy(slotname, prev_slotname, NAMEDATALEN);
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1397,7 +1512,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1405,11 +1522,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+
+	/*
+	 * Refresh the originname in case of having non-existing origin
+	 * from previous failed sync attempts.
+	 * If that's the case, it should be removed from the catalog so far.
+	 * Then, we can continue by reusing the origin created by the current
+	 * worker instead of .
+	 */
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									MyLogicalRepWorker->relid,
+									originname,
+									sizeof(originname));
+
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					slotname, originname, MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
 	pgstat_report_stat(true);
 
 	StartTransactionCommand();
@@ -1437,48 +1574,96 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: created replication slot %s for subscription %u",
+						slotname, MyLogicalRepWorker->subid)));
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: reusing replication slot %s for relation %u in subscription %u",
+						slotname, MyLogicalRepWorker->relid,
+						MyLogicalRepWorker->subid)));
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/*
 	 * Make sure that the copy command runs as the table owner, unless the
 	 * user has opted out of that behaviour.
@@ -1537,12 +1722,18 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					originname, slotname, SUBREL_STATE_FINISHEDCOPY,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 
 copy_table_done:
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 537cd33a30..8966e02b77 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -466,8 +466,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -4504,6 +4512,9 @@ start_table_sync(XLogRecPtr *origin_startpos,
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4570,10 +4581,12 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname, worker_slot);
 
+	StartTransactionCommand();
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   originname_size);
+	CommitTransactionCommand();
 
 	set_apply_error_context_origin(originname);
 
@@ -4613,11 +4626,10 @@ run_apply_worker(WalRcvStreamOptions *options,
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 					errmsg("subscription has no replication slot set")));
 
-	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-									   originname, originname_size);
-
 	/* Setup replication origin tracking. */
 	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
 	originid = replorigin_by_name(originname, true);
 	if (!OidIsValid(originid))
 		originid = replorigin_create(originname);
@@ -4934,7 +4946,32 @@ TablesyncWorkerMain(Datum main_arg)
 			}
 
 			if (!is_table_found)
+			{
+				TimeLineID	tli;
+
+				/*
+				 * It is important to give an error if we are unable to drop the
+				 * slot, otherwise, it won't be dropped till the corresponding
+				 * subscription is dropped. So passing missing_ok = false.
+				 */
+				if (MyLogicalRepWorker->created_slot)
+				{
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+					ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+				}
+
+				/*
+				 * Drop replication origin before exiting.
+				 *
+				 * There is a chance that the user is concurrently performing refresh
+				 * for the subscription where we remove the table state and its origin
+				 * or the apply worker would have removed this origin. So passing
+				 * missing_ok = true.
+				 */
+				replorigin_drop_by_name(originname, true, false);
+
 				break;
+			}
 		}
 	}
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 1d40eebc78..7e13f59847 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -108,6 +108,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -144,6 +147,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -164,6 +169,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..185164d75e 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,16 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index a8a89dc784..31b2c41893 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -237,8 +237,9 @@ extern bool InvalidateObsoleteReplicationSlots(ReplicationSlotInvalidationCause
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index af6fd339f7..97c7d2aa50 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -239,7 +256,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
@@ -336,4 +354,7 @@ am_parallel_apply_worker(void)
 	return isParallelApplyWorker(MyLogicalRepWorker);
 }
 
+/* Invalid identifier to be used for naming replication slots */
+#define InvalidRepSlotId	0
+
 #endif							/* WORKER_INTERNAL_H */
-- 
2.25.1

--- on publisher
SELECT 'CREATE TABLE manytables_'||i||'(i int);' FROM generate_series(1, 100) g(i) \gexec
SELECT pg_create_logical_replication_slot('mysub_slot', 'pgoutput');

--- on subscriber
SELECT 'CREATE TABLE manytables_'||i||'(i int);' FROM generate_series(1, 100) g(i) \gexec

CREATE OR REPLACE PROCEDURE log_rep_test(max INTEGER) AS $$
DECLARE
    counter INTEGER := 1;
    total_duration INTERVAL := '0';
    avg_duration FLOAT := 0.0;
    start_time TIMESTAMP;
    end_time TIMESTAMP;
BEGIN
    WHILE counter <= max LOOP
        
        EXECUTE 'DROP SUBSCRIPTION IF EXISTS mysub;';

        start_time := clock_timestamp();
        EXECUTE 'CREATE SUBSCRIPTION mysub CONNECTION ''dbname=postgres port=5432'' PUBLICATION mypub WITH (create_slot=false, slot_name=''mysub_slot'');';
        COMMIT;

        WHILE EXISTS (SELECT 1 FROM pg_subscription_rel WHERE srsubstate != 'r') LOOP
            COMMIT;
        END LOOP;

        end_time := clock_timestamp();


        EXECUTE 'ALTER SUBSCRIPTION mysub DISABLE;';
        EXECUTE 'ALTER SUBSCRIPTION mysub SET (slot_name = none);';

        
        total_duration := total_duration + (end_time - start_time);
        
        counter := counter + 1;
    END LOOP;
    
    IF max > 0 THEN
        avg_duration := EXTRACT(EPOCH FROM total_duration) / max * 1000;
    END IF;
    
    RAISE NOTICE '%', avg_duration;
END;
$$ LANGUAGE plpgsql;


call log_rep_test(5);

From 6fc111ce99572adddca734810470b7de02a54882 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Mon, 5 Jun 2023 15:04:41 +0300
Subject: [PATCH v16 1/5] Refactor to split Apply and Tablesync Workers

Both apply and tablesync workers were using ApplyWorkerMain() as entry
point. As the name implies, ApplyWorkerMain() should be considered as
the main function for apply workers. Tablesync worker's path was hidden
and does not have enough in common to share the same main function with
apply worker.

Also; most of the code shared by both worker types are already combined
in LogicalRepApplyLoop(). There is no need to combine the rest in
ApplyWorkerMain() anymore.

This commit introduces TablesyncWorkerMain() as a new entry point for
tablesync workers and separates both type of workers from each other.
This aims to increase code readability and help to maintain logical
replication workers separately.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/postmaster/bgworker.c             |   3 +
 .../replication/logical/applyparallelworker.c |   2 +-
 src/backend/replication/logical/launcher.c    |  32 +-
 src/backend/replication/logical/tablesync.c   |   2 +-
 src/backend/replication/logical/worker.c      | 388 +++++++++++-------
 src/include/replication/logicalworker.h       |   1 +
 src/include/replication/worker_internal.h     |   4 +-
 7 files changed, 270 insertions(+), 162 deletions(-)

diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c
index 5b4bd71694..505e38376c 100644
--- a/src/backend/postmaster/bgworker.c
+++ b/src/backend/postmaster/bgworker.c
@@ -131,6 +131,9 @@ static const struct
 	},
 	{
 		"ParallelApplyWorkerMain", ParallelApplyWorkerMain
+	},
+	{
+		"TablesyncWorkerMain", TablesyncWorkerMain
 	}
 };
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 6fb96148f4..1d4e83c4c1 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -942,7 +942,7 @@ ParallelApplyWorkerMain(Datum main_arg)
 	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
 		MyLogicalRepWorker->reply_time = 0;
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 542af7d863..e231fa7f95 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -459,24 +459,30 @@ retry:
 	snprintf(bgw.bgw_library_name, MAXPGPATH, "postgres");
 
 	if (is_parallel_apply_worker)
+	{
 		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ParallelApplyWorkerMain");
-	else
-		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
-
-	if (OidIsValid(relid))
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
-				 "logical replication worker for subscription %u sync %u", subid, relid);
-	else if (is_parallel_apply_worker)
+				 "logical replication parallel apply worker for subscription %u",
+				 subid);
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication parallel worker");
+	}
+	else if (OidIsValid(relid))
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "TablesyncWorkerMain");
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
-				 "logical replication parallel apply worker for subscription %u", subid);
+				 "logical replication tablesync worker for subscription %u sync %u",
+				 subid,
+				 relid);
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication tablesync worker");
+	}
 	else
+	{
+		snprintf(bgw.bgw_function_name, BGW_MAXLEN, "ApplyWorkerMain");
 		snprintf(bgw.bgw_name, BGW_MAXLEN,
-				 "logical replication apply worker for subscription %u", subid);
-
-	if (is_parallel_apply_worker)
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication parallel worker");
-	else
-		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication worker");
+				 "logical replication apply worker for subscription %u",
+				 subid);
+		snprintf(bgw.bgw_type, BGW_MAXLEN, "logical replication apply worker");
+	}
 
 	bgw.bgw_restart_time = BGW_NEVER_RESTART;
 	bgw.bgw_notify_pid = MyProcPid;
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 6d461654ab..8125bbd170 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -131,7 +131,7 @@ static StringInfo copybuf = NULL;
 /*
  * Exit routine for synchronization worker.
  */
-static void
+void
 pg_attribute_noreturn()
 finish_sync_worker(void)
 {
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 0ee764d68f..96ddad356b 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -392,6 +392,9 @@ static void stream_open_file(Oid subid, TransactionId xid,
 static void stream_write_change(char action, StringInfo s);
 static void stream_open_and_write_change(TransactionId xid, char action, StringInfo s);
 static void stream_close_file(void);
+static void set_stream_options(WalRcvStreamOptions *options,
+							   char *slotname,
+							   XLogRecPtr *origin_startpos);
 
 static void send_feedback(XLogRecPtr recvpos, bool force, bool requestReply);
 
@@ -4330,6 +4333,72 @@ stream_open_and_write_change(TransactionId xid, char action, StringInfo s)
 	stream_stop_internal(xid);
 }
 
+ /* set_stream_options
+  * 	Set logical replication streaming options.
+  *
+  * This function sets streaming options including replication slot name and
+  * origin start position. Workers need these options for logical replication.
+  */
+static void
+set_stream_options(WalRcvStreamOptions *options,
+				   char *slotname,
+				   XLogRecPtr *origin_startpos)
+{
+	int			server_version;
+
+	options->logical = true;
+	options->startpoint = *origin_startpos;
+	options->slotname = slotname;
+
+	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+	options->proto.logical.proto_version =
+		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
+		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+		LOGICALREP_PROTO_VERSION_NUM;
+
+	options->proto.logical.publication_names = MySubscription->publications;
+	options->proto.logical.binary = MySubscription->binary;
+
+	/*
+	 * Assign the appropriate option value for streaming option according to
+	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 */
+	if (server_version >= 160000 &&
+		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
+	{
+		options->proto.logical.streaming_str = "parallel";
+		MyLogicalRepWorker->parallel_apply = true;
+	}
+	else if (server_version >= 140000 &&
+			 MySubscription->stream != LOGICALREP_STREAM_OFF)
+	{
+		options->proto.logical.streaming_str = "on";
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+	else
+	{
+		options->proto.logical.streaming_str = NULL;
+		MyLogicalRepWorker->parallel_apply = false;
+	}
+
+	options->proto.logical.twophase = false;
+	options->proto.logical.origin = pstrdup(MySubscription->origin);
+
+	/*
+	 * Even when the two_phase mode is requested by the user, it remains as
+	 * the tri-state PENDING until all tablesyncs have reached READY state.
+	 * Only then, can it become ENABLED.
+	 *
+	 * Note: If the subscription has no tables then leave the state as
+	 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
+	 * work.
+	 */
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+		options->proto.logical.twophase = true;
+}
+
 /*
  * Cleanup the memory for subxacts and reset the related variables.
  */
@@ -4433,7 +4502,8 @@ start_apply(XLogRecPtr origin_startpos)
 			 * idle state.
 			 */
 			AbortOutOfAnyTransaction();
-			pgstat_report_subscription_error(MySubscription->oid, !am_tablesync_worker());
+			pgstat_report_subscription_error(MySubscription->oid,
+											 !am_tablesync_worker());
 
 			PG_RE_THROW();
 		}
@@ -4442,13 +4512,133 @@ start_apply(XLogRecPtr origin_startpos)
 }
 
 /*
- * Common initialization for leader apply worker and parallel apply worker.
+ * Runs the tablesync worker.
+ * It starts syncing tables. After a successful sync, sets streaming options
+ * and starts streaming to catchup.
+ */
+static void
+run_tablesync_worker(WalRcvStreamOptions *options,
+					 char *slotname,
+					 char *originname,
+					 int originname_size,
+					 XLogRecPtr *origin_startpos)
+{
+	/* Start table synchronization. */
+	start_table_sync(origin_startpos, &slotname);
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									   MyLogicalRepWorker->relid,
+									   originname,
+									   originname_size);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	/* Start applying changes to catchup. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Runs the leader apply worker.
+ * It sets up replication origin, streaming options and then starts streaming.
+ */
+static void
+run_apply_worker(WalRcvStreamOptions *options,
+				 char *slotname,
+				 char *originname,
+				 int originname_size,
+				 XLogRecPtr *origin_startpos)
+{
+	RepOriginId originid;
+	TimeLineID	startpointTLI;
+	char	   *err;
+	bool		must_use_password;
+
+	slotname = MySubscription->slotname;
+
+	/*
+	 * This shouldn't happen if the subscription is enabled, but guard
+	 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
+	 * crash if slot is NULL.)
+	 */
+	if (!slotname)
+		ereport(ERROR,
+				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
+					errmsg("subscription has no replication slot set")));
+
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
+
+	/* Setup replication origin tracking. */
+	StartTransactionCommand();
+	originid = replorigin_by_name(originname, true);
+	if (!OidIsValid(originid))
+		originid = replorigin_create(originname);
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+	*origin_startpos = replorigin_session_get_progress(false);
+
+	/* Is the use of a password mandatory? */
+	must_use_password = MySubscription->passwordrequired &&
+		!superuser_arg(MySubscription->owner);
+
+	/* Note that the superuser_arg call can access the DB */
+	CommitTransactionCommand();
+
+	LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
+											must_use_password,
+											MySubscription->name, &err);
+
+	if (LogRepWorkerWalRcvConn == NULL)
+		ereport(ERROR,
+				(errcode(ERRCODE_CONNECTION_FAILURE),
+					errmsg("could not connect to the publisher: %s", err)));
+
+	/*
+	 * We don't really use the output identify_system for anything but it
+	 * does some initializations on the upstream so let's still call it.
+	 */
+	(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
+
+	set_apply_error_context_origin(originname);
+
+	set_stream_options(options, slotname, origin_startpos);
+
+	walrcv_startstreaming(LogRepWorkerWalRcvConn, options);
+
+	if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
+		AllTablesyncsReady())
+	{
+		StartTransactionCommand();
+		UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
+		MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
+		CommitTransactionCommand();
+	}
+
+	ereport(DEBUG1,
+			(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
+							 MySubscription->name,
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
+							 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
+							 "?")));
+
+	/* Run the main loop. */
+	start_apply(*origin_startpos);
+}
+
+/*
+ * Common initialization for logical replication workers; leader apply worker,
+ * parallel apply worker and tablesync worker.
  *
  * Initialize the database connection, in-memory subscription and necessary
  * config options.
  */
 void
-InitializeApplyWorker(void)
+InitializeLogRepWorker(void)
 {
 	MemoryContext oldctx;
 
@@ -4512,7 +4702,8 @@ InitializeApplyWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+						get_worker_name(),
 						MySubscription->name,
 						get_rel_name(MyLogicalRepWorker->relid))));
 	else
@@ -4533,7 +4724,6 @@ ApplyWorkerMain(Datum main_arg)
 	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
 	char	   *myslotname = NULL;
 	WalRcvStreamOptions options;
-	int			server_version;
 
 	InitializingApplyWorker = true;
 
@@ -4557,7 +4747,7 @@ ApplyWorkerMain(Datum main_arg)
 	/* Load the libpq-specific functions */
 	load_file("libpqwalreceiver", false);
 
-	InitializeApplyWorker();
+	InitializeLogRepWorker();
 
 	InitializingApplyWorker = false;
 
@@ -4565,71 +4755,6 @@ ApplyWorkerMain(Datum main_arg)
 	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
 		 MySubscription->conninfo);
 
-	if (am_tablesync_worker())
-	{
-		start_table_sync(&origin_startpos, &myslotname);
-
-		ReplicationOriginNameForLogicalRep(MySubscription->oid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
-		set_apply_error_context_origin(originname);
-	}
-	else
-	{
-		/* This is the leader apply worker */
-		RepOriginId originid;
-		TimeLineID	startpointTLI;
-		char	   *err;
-		bool		must_use_password;
-
-		myslotname = MySubscription->slotname;
-
-		/*
-		 * This shouldn't happen if the subscription is enabled, but guard
-		 * against DDL bugs or manual catalog changes.  (libpqwalreceiver will
-		 * crash if slot is NULL.)
-		 */
-		if (!myslotname)
-			ereport(ERROR,
-					(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
-					 errmsg("subscription has no replication slot set")));
-
-		/* Setup replication origin tracking. */
-		StartTransactionCommand();
-		ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-										   originname, sizeof(originname));
-		originid = replorigin_by_name(originname, true);
-		if (!OidIsValid(originid))
-			originid = replorigin_create(originname);
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
-		origin_startpos = replorigin_session_get_progress(false);
-
-		/* Is the use of a password mandatory? */
-		must_use_password = MySubscription->passwordrequired &&
-			!superuser_arg(MySubscription->owner);
-
-		/* Note that the superuser_arg call can access the DB */
-		CommitTransactionCommand();
-
-		LogRepWorkerWalRcvConn = walrcv_connect(MySubscription->conninfo, true,
-												must_use_password,
-												MySubscription->name, &err);
-		if (LogRepWorkerWalRcvConn == NULL)
-			ereport(ERROR,
-					(errcode(ERRCODE_CONNECTION_FAILURE),
-					 errmsg("could not connect to the publisher: %s", err)));
-
-		/*
-		 * We don't really use the output identify_system for anything but it
-		 * does some initializations on the upstream so let's still call it.
-		 */
-		(void) walrcv_identify_system(LogRepWorkerWalRcvConn, &startpointTLI);
-
-		set_apply_error_context_origin(originname);
-	}
-
 	/*
 	 * Setup callback for syscache so that we know when something changes in
 	 * the subscription relation state.
@@ -4638,92 +4763,63 @@ ApplyWorkerMain(Datum main_arg)
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	/* Build logical replication streaming options. */
-	options.logical = true;
-	options.startpoint = origin_startpos;
-	options.slotname = myslotname;
+	/* This is leader apply worker */
+	run_apply_worker(&options, myslotname, originname, sizeof(originname), &origin_startpos);
 
-	server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
-	options.proto.logical.proto_version =
-		server_version >= 160000 ? LOGICALREP_PROTO_STREAM_PARALLEL_VERSION_NUM :
-		server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
-		server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
-		LOGICALREP_PROTO_VERSION_NUM;
+	proc_exit(0);
+}
 
-	options.proto.logical.publication_names = MySubscription->publications;
-	options.proto.logical.binary = MySubscription->binary;
+/* Logical Replication Tablesync worker entry point */
+void
+TablesyncWorkerMain(Datum main_arg)
+{
+	int			worker_slot = DatumGetInt32(main_arg);
+	char		originname[NAMEDATALEN];
+	XLogRecPtr	origin_startpos = InvalidXLogRecPtr;
+	char	   *myslotname = NULL;
+	WalRcvStreamOptions options;
+
+	/* Attach to slot */
+	logicalrep_worker_attach(worker_slot);
+
+	/* Setup signal handling */
+	pqsignal(SIGHUP, SignalHandlerForConfigReload);
+	pqsignal(SIGTERM, die);
+	BackgroundWorkerUnblockSignals();
 
 	/*
-	 * Assign the appropriate option value for streaming option according to
-	 * the 'streaming' mode and the publisher's ability to support that mode.
+	 * We don't currently need any ResourceOwner in a walreceiver process, but
+	 * if we did, we could call CreateAuxProcessResourceOwner here.
 	 */
-	if (server_version >= 160000 &&
-		MySubscription->stream == LOGICALREP_STREAM_PARALLEL)
-	{
-		options.proto.logical.streaming_str = "parallel";
-		MyLogicalRepWorker->parallel_apply = true;
-	}
-	else if (server_version >= 140000 &&
-			 MySubscription->stream != LOGICALREP_STREAM_OFF)
-	{
-		options.proto.logical.streaming_str = "on";
-		MyLogicalRepWorker->parallel_apply = false;
-	}
-	else
-	{
-		options.proto.logical.streaming_str = NULL;
-		MyLogicalRepWorker->parallel_apply = false;
-	}
 
-	options.proto.logical.twophase = false;
-	options.proto.logical.origin = pstrdup(MySubscription->origin);
+	/* Initialise stats to a sanish value */
+	MyLogicalRepWorker->last_send_time = MyLogicalRepWorker->last_recv_time =
+		MyLogicalRepWorker->reply_time = GetCurrentTimestamp();
 
-	if (!am_tablesync_worker())
-	{
-		/*
-		 * Even when the two_phase mode is requested by the user, it remains
-		 * as the tri-state PENDING until all tablesyncs have reached READY
-		 * state. Only then, can it become ENABLED.
-		 *
-		 * Note: If the subscription has no tables then leave the state as
-		 * PENDING, which allows ALTER SUBSCRIPTION ... REFRESH PUBLICATION to
-		 * work.
-		 */
-		if (MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING &&
-			AllTablesyncsReady())
-		{
-			/* Start streaming with two_phase enabled */
-			options.proto.logical.twophase = true;
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
+	/* Load the libpq-specific functions */
+	load_file("libpqwalreceiver", false);
 
-			StartTransactionCommand();
-			UpdateTwoPhaseState(MySubscription->oid, LOGICALREP_TWOPHASE_STATE_ENABLED);
-			MySubscription->twophasestate = LOGICALREP_TWOPHASE_STATE_ENABLED;
-			CommitTransactionCommand();
-		}
-		else
-		{
-			walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-		}
+	InitializeLogRepWorker();
 
-		ereport(DEBUG1,
-				(errmsg_internal("logical replication apply worker for subscription \"%s\" two_phase is %s",
-								 MySubscription->name,
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_DISABLED ? "DISABLED" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_PENDING ? "PENDING" :
-								 MySubscription->twophasestate == LOGICALREP_TWOPHASE_STATE_ENABLED ? "ENABLED" :
-								 "?")));
-	}
-	else
-	{
-		/* Start normal logical streaming replication. */
-		walrcv_startstreaming(LogRepWorkerWalRcvConn, &options);
-	}
+	/* Connect to the origin and start the replication. */
+	elog(DEBUG1, "connecting to publisher using connection string \"%s\"",
+		 MySubscription->conninfo);
 
-	/* Run the main loop. */
-	start_apply(origin_startpos);
+	/*
+	 * Setup callback for syscache so that we know when something changes in
+	 * the subscription relation state.
+	 */
+	CacheRegisterSyscacheCallback(SUBSCRIPTIONRELMAP,
+								  invalidate_syncing_table_states,
+								  (Datum) 0);
 
-	proc_exit(0);
+	run_tablesync_worker(&options,
+						 myslotname,
+						 originname,
+						 sizeof(originname),
+						 &origin_startpos);
+
+	finish_sync_worker();
 }
 
 /*
diff --git a/src/include/replication/logicalworker.h b/src/include/replication/logicalworker.h
index 39588da79f..bbd71d0b42 100644
--- a/src/include/replication/logicalworker.h
+++ b/src/include/replication/logicalworker.h
@@ -18,6 +18,7 @@ extern PGDLLIMPORT volatile sig_atomic_t ParallelApplyMessagePending;
 
 extern void ApplyWorkerMain(Datum main_arg);
 extern void ParallelApplyWorkerMain(Datum main_arg);
+extern void TablesyncWorkerMain(Datum main_arg);
 
 extern bool IsLogicalWorker(void);
 extern bool IsLogicalParallelApplyWorker(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 343e781896..7aba034774 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -265,7 +265,7 @@ extern void maybe_reread_subscription(void);
 
 extern void stream_cleanup_files(Oid subid, TransactionId xid);
 
-extern void InitializeApplyWorker(void);
+extern void InitializeLogRepWorker(void);
 
 extern void store_flush_position(XLogRecPtr remote_lsn, XLogRecPtr local_lsn);
 
@@ -307,6 +307,8 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
+extern void finish_sync_worker(void);
+
 static inline bool
 am_tablesync_worker(void)
 {
-- 
2.27.0

From db3e8e2d7aadea79126c5816bce8b06dc82f33c2 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Tue, 4 Jul 2023 22:04:46 +0300
Subject: [PATCH v16 2/5] Reuse Tablesync Workers

This commit allows reusing tablesync workers for syncing more than one
table sequentially during their lifetime, instead of exiting after
only syncing one table.

Before this commit, tablesync workers were capable of syncing only one
table. For each table, a new sync worker was launched and that worker would
exit when done processing the table.

Now, tablesync workers are not limited to processing only one
table. When done, they can move to processing another table in
the same subscription.

If there is a table that needs to be synced, an available tablesync
worker picks up that table and syncs it. Each tablesync worker
continues to pick new tables to sync until there are no tables left
requiring synchronization. If there was no available worker to
process the table, then a new tablesync worker will be launched,
provided the number of tablesync workers for the subscription does not
exceed max_sync_workers_per_subscription.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 src/backend/replication/logical/launcher.c  |   1 +
 src/backend/replication/logical/tablesync.c |  50 ++++++--
 src/backend/replication/logical/worker.c    | 121 ++++++++++++++++++--
 src/include/replication/worker_internal.h   |   7 ++
 4 files changed, 158 insertions(+), 21 deletions(-)

diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index e231fa7f95..72e5ef8a78 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -440,6 +440,7 @@ retry:
 	worker->stream_fileset = NULL;
 	worker->leader_pid = is_parallel_apply_worker ? MyProcPid : InvalidPid;
 	worker->parallel_apply = is_parallel_apply_worker;
+	worker->is_sync_completed = false;
 	worker->last_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->last_send_time);
 	TIMESTAMP_NOBEGIN(worker->last_recv_time);
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 8125bbd170..605c5bd4ec 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -129,15 +129,14 @@ static bool FetchTableStates(bool *started_tx);
 static StringInfo copybuf = NULL;
 
 /*
- * Exit routine for synchronization worker.
+ * Prepares the synchronization worker for reuse or exit.
  */
 void
-pg_attribute_noreturn()
-finish_sync_worker(void)
+clean_sync_worker(void)
 {
 	/*
-	 * Commit any outstanding transaction. This is the usual case, unless
-	 * there was nothing to do for the table.
+	 * Commit any outstanding transaction. This is the usual case, unless there
+	 * was nothing to do for the table.
 	 */
 	if (IsTransactionState())
 	{
@@ -145,19 +144,38 @@ finish_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders
+	 */
+	if (LogRepWorkerWalRcvConn != NULL)
+	{
+		walrcv_disconnect(LogRepWorkerWalRcvConn);
+		LogRepWorkerWalRcvConn = NULL;
+	}
+
+	/* Find the leader apply worker and signal it. */
+	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
+}
+
+/*
+ * Exit routine for synchronization worker.
+ */
+void
+pg_attribute_noreturn()
+finish_sync_worker(void)
+{
+	clean_sync_worker();
+
 	/* And flush all writes. */
 	XLogFlush(GetXLogWriteRecPtr());
 
 	StartTransactionCommand();
 	ereport(LOG,
-			(errmsg("logical replication table synchronization worker for subscription \"%s\", table \"%s\" has finished",
-					MySubscription->name,
-					get_rel_name(MyLogicalRepWorker->relid))));
+			(errmsg("logical replication table synchronization worker for subscription \"%s\" has finished",
+					MySubscription->name)));
 	CommitTransactionCommand();
 
-	/* Find the leader apply worker and signal it. */
-	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
-
 	/* Stop gracefully */
 	proc_exit(0);
 }
@@ -379,7 +397,15 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 */
 		replorigin_drop_by_name(originname, true, false);
 
-		finish_sync_worker();
+		/* Sync worker has completed synchronization of the current table. */
+		MyLogicalRepWorker->is_sync_completed = true;
+
+		ereport(LOG,
+				(errmsg("logical replication table synchronization worker for subscription \"%s\", relation \"%s\" with relid %u has finished",
+						MySubscription->name,
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
+		CommitTransactionCommand();
 	}
 	else
 		SpinLockRelease(&MyLogicalRepWorker->relmutex);
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 96ddad356b..eae561db05 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3626,6 +3626,20 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 					MemoryContextReset(ApplyMessageContext);
 				}
 
+				/*
+				 * apply_dispatch() may have gone into apply_handle_commit()
+				 * which can call process_syncing_tables_for_sync.
+				 *
+				 * process_syncing_tables_for_sync decides whether the sync of
+				 * the current table is completed. If it is completed,
+				 * streaming must be already ended. So, we can break the loop.
+				 */
+				if (MyLogicalRepWorker->is_sync_completed)
+				{
+					endofstream = true;
+					break;
+				}
+
 				len = walrcv_receive(LogRepWorkerWalRcvConn, &buf, &fd);
 			}
 		}
@@ -3645,6 +3659,15 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 			/* Process any table synchronization changes. */
 			process_syncing_tables(last_received);
+
+			/*
+			 * If is_sync_completed is true, this means that the tablesync
+			 * worker is done with synchronization. Streaming has already been
+			 * ended by process_syncing_tables_for_sync. We should move to the
+			 * next table if needed, or exit.
+			 */
+			if (MyLogicalRepWorker->is_sync_completed)
+				endofstream = true;
 		}
 
 		/* Cleanup the memory. */
@@ -3747,8 +3770,12 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 	error_context_stack = errcallback.previous;
 	apply_error_context_stack = error_context_stack;
 
-	/* All done */
-	walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	/*
+	 * Tablesync workers should end streaming before exiting the main loop to
+	 * drop replication slot. Only end streaming here for apply workers.
+	 */
+	if (!am_tablesync_worker())
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
 }
 
 /*
@@ -4523,6 +4550,8 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 					 int originname_size,
 					 XLogRecPtr *origin_startpos)
 {
+	MyLogicalRepWorker->is_sync_completed = false;
+
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname);
 
@@ -4702,10 +4731,11 @@ InitializeLogRepWorker(void)
 
 	if (am_tablesync_worker())
 		ereport(LOG,
-				(errmsg("%s for subscription \"%s\", table \"%s\" has started",
+				(errmsg("%s for subscription \"%s\", table \"%s\" with relid %u has started",
 						get_worker_name(),
 						MySubscription->name,
-						get_rel_name(MyLogicalRepWorker->relid))));
+						get_rel_name(MyLogicalRepWorker->relid),
+						MyLogicalRepWorker->relid)));
 	else
 		ereport(LOG,
 		/* translator: first %s is the name of logical replication worker */
@@ -4813,11 +4843,84 @@ TablesyncWorkerMain(Datum main_arg)
 								  invalidate_syncing_table_states,
 								  (Datum) 0);
 
-	run_tablesync_worker(&options,
-						 myslotname,
-						 originname,
-						 sizeof(originname),
-						 &origin_startpos);
+	/*
+	 * The loop where worker does its job. It loops until there is no relation
+	 * left to sync.
+	 */
+	for (;;)
+	{
+		List	   *rstates;
+		ListCell   *lc;
+		bool 	is_table_found = false;
+
+		run_tablesync_worker(&options,
+							 myslotname,
+							 originname,
+							 sizeof(originname),
+							 &origin_startpos);
+
+		if (IsTransactionState())
+			CommitTransactionCommand();
+
+		if (MyLogicalRepWorker->is_sync_completed)
+		{
+			/* This transaction will be committed by clean_sync_worker. */
+			StartTransactionCommand();
+
+			/*
+			 * Check if there is any table whose relation state is still INIT.
+			 * If a table in INIT state is found, the worker will not be
+			 * finished, it will be reused instead.
+			 */
+			rstates = GetSubscriptionRelations(MySubscription->oid, true);
+
+			foreach(lc, rstates)
+			{
+				SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
+
+				if (rstate->state == SUBREL_STATE_SYNCDONE)
+					continue;
+
+				/*
+				 * Take exclusive lock to prevent any other sync worker from
+				 * picking the same table.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_EXCLUSIVE);
+
+				/*
+				 * Pick the table for the next run if it is not already picked
+				 * up by another worker.
+				 */
+				if (!logicalrep_worker_find(MySubscription->oid, rstate->relid, false))
+				{
+					/* Update worker state for the next table */
+					MyLogicalRepWorker->relid = rstate->relid;
+					MyLogicalRepWorker->relstate = rstate->state;
+					MyLogicalRepWorker->relstate_lsn = rstate->lsn;
+					LWLockRelease(LogicalRepWorkerLock);
+
+					/* Found a table for next iteration */
+					is_table_found = true;
+					clean_sync_worker();
+
+					StartTransactionCommand();
+					ereport(LOG,
+							(errmsg("%s for subscription \"%s\" has moved to sync table \"%s\" with relid %u.",
+									get_worker_name(),
+									MySubscription->name,
+									get_rel_name(MyLogicalRepWorker->relid),
+									MyLogicalRepWorker->relid)));
+					CommitTransactionCommand();
+
+					break;
+				}
+				LWLockRelease(LogicalRepWorkerLock);
+			}
+
+			if (!is_table_found)
+				break;
+		}
+	}
 
 	finish_sync_worker();
 }
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 7aba034774..1e9f8e6e72 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -56,6 +56,12 @@ typedef struct LogicalRepWorker
 	XLogRecPtr	relstate_lsn;
 	slock_t		relmutex;
 
+	/*
+	 * Indicates whether tablesync worker has completed sycning its assigned
+	 * table. If true, no need to continue with that table.
+	 */
+	bool		is_sync_completed;
+
 	/*
 	 * Used to create the changes and subxact files for the streaming
 	 * transactions.  Upon the arrival of the first streaming transaction or
@@ -308,6 +314,7 @@ extern void pa_xact_finish(ParallelApplyWorkerInfo *winfo,
 #define isParallelApplyWorker(worker) ((worker)->leader_pid != InvalidPid)
 
 extern void finish_sync_worker(void);
+extern void clean_sync_worker(void);
 
 static inline bool
 am_tablesync_worker(void)
-- 
2.27.0

From b0f3066e5d592c603aee8f89ff4171f2b2a8e7f6 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Tue, 4 Jul 2023 22:13:52 +0300
Subject: [PATCH v16 3/5] reuse connection when tablesync workers change the
 target

---
 src/backend/replication/logical/tablesync.c | 53 ++++++++++++++-------
 src/backend/replication/logical/worker.c    | 30 +++++++-----
 src/backend/replication/walsender.c         |  6 +++
 src/include/replication/worker_internal.h   |  3 +-
 4 files changed, 61 insertions(+), 31 deletions(-)

diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index 605c5bd4ec..f042d9ae00 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -144,16 +144,6 @@ clean_sync_worker(void)
 		pgstat_report_stat(true);
 	}
 
-	/*
-	 * Disconnect from publisher. Otherwise reused sync workers causes
-	 * exceeding max_wal_senders
-	 */
-	if (LogRepWorkerWalRcvConn != NULL)
-	{
-		walrcv_disconnect(LogRepWorkerWalRcvConn);
-		LogRepWorkerWalRcvConn = NULL;
-	}
-
 	/* Find the leader apply worker and signal it. */
 	logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
 }
@@ -167,6 +157,16 @@ finish_sync_worker(void)
 {
 	clean_sync_worker();
 
+	/*
+	 * Disconnect from publisher. Otherwise reused sync workers causes
+	 * exceeding max_wal_senders.
+	 */
+	if (LogRepWorkerWalRcvConn != NULL)
+	{
+		walrcv_disconnect(LogRepWorkerWalRcvConn);
+		LogRepWorkerWalRcvConn = NULL;
+	}
+
 	/* And flush all writes. */
 	XLogFlush(GetXLogWriteRecPtr());
 
@@ -1268,7 +1268,7 @@ ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
  * The returned slot name is palloc'ed in current memory context.
  */
 char *
-LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
+LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 {
 	char	   *slotname;
 	char	   *err;
@@ -1321,14 +1321,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos)
 									NAMEDATALEN);
 
 	/*
-	 * Here we use the slot name instead of the subscription name as the
-	 * application_name, so that it is different from the leader apply worker,
-	 * so that synchronous replication can distinguish them.
+	 * Connect to publisher if not yet. The application_name must be also
+	 * different from the leader apply worker because synchronous replication
+	 * must distinguish them.
 	 */
-	LogRepWorkerWalRcvConn =
-		walrcv_connect(MySubscription->conninfo, true,
-					   must_use_password,
-					   slotname, &err);
+	if (LogRepWorkerWalRcvConn == NULL)
+	{
+		char application_name[NAMEDATALEN];
+
+		/*
+		 * FIXME: set appropriate application_name. Previously, the slot name
+		 * was used becasue the lifetime of the tablesync worker was same as
+		 * that, but now the tablesync worker handles many slots during the
+		 * synchronization so that it is not suitable. So what should be?
+		 * Note that if the tablesync worker starts to reuse the replication
+		 * slot during synchronization, we should use the slot name as
+		 * application_name again.
+		 */
+		snprintf(application_name, NAMEDATALEN, "pg_%u_sync_%i",
+				 MySubscription->oid, worker_slot);
+		LogRepWorkerWalRcvConn =
+			walrcv_connect(MySubscription->conninfo, true,
+						   must_use_password,
+						   application_name, &err);
+	}
+
 	if (LogRepWorkerWalRcvConn == NULL)
 		ereport(ERROR,
 				(errcode(ERRCODE_CONNECTION_FAILURE),
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index eae561db05..ca663445c1 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -3500,19 +3500,21 @@ LogicalRepApplyLoop(XLogRecPtr last_received)
 
 	/*
 	 * Init the ApplyMessageContext which we clean up after each replication
-	 * protocol message.
+	 * protocol message, if needed.
 	 */
-	ApplyMessageContext = AllocSetContextCreate(ApplyContext,
-												"ApplyMessageContext",
-												ALLOCSET_DEFAULT_SIZES);
+	if (!ApplyMessageContext)
+		ApplyMessageContext = AllocSetContextCreate(ApplyContext,
+													"ApplyMessageContext",
+													ALLOCSET_DEFAULT_SIZES);
 
 	/*
 	 * This memory context is used for per-stream data when the streaming mode
 	 * is enabled. This context is reset on each stream stop.
 	 */
-	LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
-													"LogicalStreamingContext",
-													ALLOCSET_DEFAULT_SIZES);
+	if (!LogicalStreamingContext)
+		LogicalStreamingContext = AllocSetContextCreate(ApplyContext,
+														"LogicalStreamingContext",
+														ALLOCSET_DEFAULT_SIZES);
 
 	/* mark as idle, before starting to loop */
 	pgstat_report_activity(STATE_IDLE, NULL);
@@ -4468,7 +4470,9 @@ TwoPhaseTransactionGid(Oid subid, TransactionId xid, char *gid, int szgid)
  * are not repeatable.
  */
 static void
-start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
+start_table_sync(XLogRecPtr *origin_startpos,
+				 char **myslotname,
+				 int worker_slot)
 {
 	char	   *syncslotname = NULL;
 
@@ -4477,7 +4481,7 @@ start_table_sync(XLogRecPtr *origin_startpos, char **myslotname)
 	PG_TRY();
 	{
 		/* Call initial sync. */
-		syncslotname = LogicalRepSyncTableStart(origin_startpos);
+		syncslotname = LogicalRepSyncTableStart(origin_startpos, worker_slot);
 	}
 	PG_CATCH();
 	{
@@ -4548,12 +4552,13 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 					 char *slotname,
 					 char *originname,
 					 int originname_size,
-					 XLogRecPtr *origin_startpos)
+					 XLogRecPtr *origin_startpos,
+					 int worker_slot)
 {
 	MyLogicalRepWorker->is_sync_completed = false;
 
 	/* Start table synchronization. */
-	start_table_sync(origin_startpos, &slotname);
+	start_table_sync(origin_startpos, &slotname, worker_slot);
 
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
@@ -4857,7 +4862,8 @@ TablesyncWorkerMain(Datum main_arg)
 							 myslotname,
 							 originname,
 							 sizeof(originname),
-							 &origin_startpos);
+							 &origin_startpos,
+							 worker_slot);
 
 		if (IsTransactionState())
 			CommitTransactionCommand();
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index cedadb0036..06adcf6f01 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1827,6 +1827,12 @@ exec_replication_command(const char *cmd_string)
 				set_ps_display(cmdtag);
 				PreventInTransactionBlock(true, cmdtag);
 
+				/*
+				 * Initialize the flag again because this streaming may be
+				 * second time.
+				 */
+				streamingDoneSending = streamingDoneReceiving = false;
+
 				if (cmd->kind == REPLICATION_KIND_PHYSICAL)
 					StartReplication(cmd);
 				else
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 1e9f8e6e72..af6fd339f7 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -249,7 +249,8 @@ extern int	logicalrep_sync_worker_count(Oid subid);
 
 extern void ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 											   char *originname, Size szoriginname);
-extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos);
+extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos,
+									  int worker_slot);
 
 extern bool AllTablesyncsReady(void);
 extern void UpdateTwoPhaseState(Oid suboid, char new_state);
-- 
2.27.0

From 8cb6298a45137f853a07470f56c07296d676692a Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 13 Oct 2022 17:05:45 +0300
Subject: [PATCH v16 4/5] Add replication protocol cmd to create a snapshot

Introduced CREATE_REPLICATION_SNAPSHOT to be able to create and use a
snapshot without creating a new replication slot, but by using an
existing slot.

CREATE_REPLICATION_SNAPSHOT simply does what CREATE_REPLICATION_SLOT does
without creating a new replication slot.

CREATE_REPLICATION_SNAPSHOT command imports the snapshot into the current
transaction and returns consistent_point. The changes earlier than the
consistent_point will be applied by importing the snapshot. All changes
later than the consistent_point will be available to be consumed from
the replication slot.

This is useful for reusing replication slots in logical replication.
Otherwise, tablesync workers cannot start from a consistent point to copy
a relation and then apply changes by consuming from replication slot.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/protocol.sgml                    |  31 ++++++
 .../libpqwalreceiver/libpqwalreceiver.c       |  69 +++++++++++-
 src/backend/replication/logical/logical.c     |  40 ++++++-
 .../replication/logical/logicalfuncs.c        |   1 +
 src/backend/replication/repl_gram.y           |  18 ++-
 src/backend/replication/repl_scanner.l        |   2 +
 src/backend/replication/slotfuncs.c           |   1 +
 src/backend/replication/walsender.c           | 104 +++++++++++++++++-
 src/include/nodes/replnodes.h                 |  11 ++
 src/include/replication/logical.h             |   1 +
 src/include/replication/walreceiver.h         |  13 +++
 src/tools/pgindent/typedefs.list              |   2 +
 12 files changed, 289 insertions(+), 4 deletions(-)

diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index b11d9a6ba3..f7d5acee93 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -2595,6 +2595,37 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
      </listitem>
     </varlistentry>
 
+    <varlistentry id="protocol-replication-replication-slot-snapshot">
+     <term><literal>CREATE_REPLICATION_SNAPSHOT</literal> <replaceable class="parameter">slot_name</replaceable> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
+      <indexterm><primary>CREATE_REPLICATION_SNAPSHOT</primary></indexterm>
+     </term>
+     <listitem>
+      <para>
+       Creates a snapshot including all the changes from the replication slot until
+       the point at which the replication slot becomes consistent. Then the snapshot
+       is used in the current transaction. This command is currently only supported
+       for logical replication slots.
+      </para>
+
+      <para>
+       In response to this command, the server will return a one-row result set,
+       containing the following field:
+       <variablelist>
+        <varlistentry>
+         <term><literal>consistent_point</literal> (<type>text</type>)</term>
+         <listitem>
+          <para>
+           The WAL location at which the slot became consistent.  This is the
+           earliest location from which streaming can start on this replication
+           slot.
+          </para>
+         </listitem>
+        </varlistentry>
+       </variablelist>
+      </para>
+     </listitem>
+    </varlistentry>
+
     <varlistentry id="protocol-replication-base-backup" xreflabel="BASE_BACKUP">
      <term><literal>BASE_BACKUP</literal> [ ( <replaceable class="parameter">option</replaceable> [, ...] ) ]
       <indexterm><primary>BASE_BACKUP</primary></indexterm>
diff --git a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
index dc9c5c82d9..17358bf552 100644
--- a/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
+++ b/src/backend/replication/libpqwalreceiver/libpqwalreceiver.c
@@ -82,6 +82,8 @@ static WalRcvExecResult *libpqrcv_exec(WalReceiverConn *conn,
 									   const int nRetTypes,
 									   const Oid *retTypes);
 static void libpqrcv_disconnect(WalReceiverConn *conn);
+static void libpqrcv_slot_snapshot(WalReceiverConn *conn, char *slotname,
+								   const WalRcvStreamOptions *options, XLogRecPtr *lsn);
 
 static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_connect = libpqrcv_connect,
@@ -98,7 +100,8 @@ static WalReceiverFunctionsType PQWalReceiverFunctions = {
 	.walrcv_create_slot = libpqrcv_create_slot,
 	.walrcv_get_backend_pid = libpqrcv_get_backend_pid,
 	.walrcv_exec = libpqrcv_exec,
-	.walrcv_disconnect = libpqrcv_disconnect
+	.walrcv_disconnect = libpqrcv_disconnect,
+	.walrcv_slot_snapshot = libpqrcv_slot_snapshot
 };
 
 /* Prototypes for private functions */
@@ -982,6 +985,70 @@ libpqrcv_create_slot(WalReceiverConn *conn, const char *slotname,
 	return snapshot;
 }
 
+/*
+ * Create a new snapshot by using an existing replication slot.
+ */
+static void
+libpqrcv_slot_snapshot(WalReceiverConn *conn,
+					   char *slotname,
+					   const WalRcvStreamOptions *options,
+					   XLogRecPtr *lsn)
+{
+	StringInfoData cmd;
+	PGresult   *res;
+	char	   *pubnames_str;
+	List	   *pubnames;
+	char	   *pubnames_literal;
+
+	initStringInfo(&cmd);
+
+	/* Build the command. */
+	appendStringInfo(&cmd, "CREATE_REPLICATION_SNAPSHOT \"%s\"", slotname);
+	appendStringInfoString(&cmd, " (");
+	appendStringInfo(&cmd, " proto_version '%u'",
+					 options->proto.logical.proto_version);
+
+	/* Add publication names. */
+	pubnames = options->proto.logical.publication_names;
+	pubnames_str = stringlist_to_identifierstr(conn->streamConn, pubnames);
+	if (!pubnames_str)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("Could not create a snapshot by replication slot %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	pubnames_literal = PQescapeLiteral(conn->streamConn, pubnames_str,
+									   strlen(pubnames_str));
+	if (!pubnames_literal)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),	/* likely guess */
+				 errmsg("Could not create a snapshot by replication slot %s",
+						pchomp(PQerrorMessage(conn->streamConn)))));
+	appendStringInfo(&cmd, ", publication_names %s", pubnames_literal);
+	PQfreemem(pubnames_literal);
+	pfree(pubnames_str);
+
+	appendStringInfoString(&cmd, " )");
+
+	/* Execute the command. */
+	res = libpqrcv_PQexec(conn->streamConn, cmd.data);
+	pfree(cmd.data);
+
+	if (PQresultStatus(res) != PGRES_TUPLES_OK)
+	{
+		PQclear(res);
+		ereport(ERROR,
+				(errcode(ERRCODE_PROTOCOL_VIOLATION),
+				 errmsg("Could not create a snapshot by replication slot \"%s\": %s",
+						slotname, pchomp(PQerrorMessage(conn->streamConn)))));
+	}
+
+	if (lsn)
+		*lsn = DatumGetLSN(DirectFunctionCall1Coll(pg_lsn_in, InvalidOid,
+												   CStringGetDatum(PQgetvalue(res, 0, 0))));
+
+	PQclear(res);
+}
+
 /*
  * Return PID of remote backend process.
  */
diff --git a/src/backend/replication/logical/logical.c b/src/backend/replication/logical/logical.c
index 41243d0187..3c11f8e1dd 100644
--- a/src/backend/replication/logical/logical.c
+++ b/src/backend/replication/logical/logical.c
@@ -476,6 +476,10 @@ CreateInitDecodingContext(const char *plugin,
  * fast_forward
  *		bypass the generation of logical changes.
  *
+ * need_full_snapshot
+ * 		if true, must obtain a snapshot able to read all tables;
+ *  	if false, one that can read only catalogs is acceptable.
+ *
  * xl_routine
  *		XLogReaderRoutine used by underlying xlogreader
  *
@@ -494,6 +498,7 @@ LogicalDecodingContext *
 CreateDecodingContext(XLogRecPtr start_lsn,
 					  List *output_plugin_options,
 					  bool fast_forward,
+					  bool need_full_snapshot,
 					  XLogReaderRoutine *xl_routine,
 					  LogicalOutputPluginWriterPrepareWrite prepare_write,
 					  LogicalOutputPluginWriterWrite do_write,
@@ -502,6 +507,7 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 	LogicalDecodingContext *ctx;
 	ReplicationSlot *slot;
 	MemoryContext old_context;
+	TransactionId xmin_horizon = InvalidTransactionId;
 
 	/* shorter lines... */
 	slot = MyReplicationSlot;
@@ -571,8 +577,40 @@ CreateDecodingContext(XLogRecPtr start_lsn,
 		start_lsn = slot->data.confirmed_flush;
 	}
 
+
+	/*
+	 * We need to determine a safe xmin horizon to start decoding from if we
+	 * want to create a snapshot too. Otherwise we would end up with a
+	 * snapshot that cannot be imported since xmin value from the snapshot may
+	 * be less than the oldest safe xmin. To avoid this call
+	 * GetOldestSafeDecodingTransactionId() to return a safe xmin value, which
+	 * can be used while exporting/importing the snapshot.
+	 *
+	 * So we have to acquire the ProcArrayLock to prevent computation of new
+	 * xmin horizons by other backends, get the safe decoding xid, and inform
+	 * the slot machinery about the new limit. Once that's done the
+	 * ProcArrayLock can be released as the slot machinery now is protecting
+	 * against vacuum.
+	 */
+	if (need_full_snapshot)
+	{
+		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
+
+		xmin_horizon = GetOldestSafeDecodingTransactionId(!need_full_snapshot);
+
+		SpinLockAcquire(&slot->mutex);
+		slot->effective_catalog_xmin = xmin_horizon;
+		slot->data.catalog_xmin = xmin_horizon;
+		slot->effective_xmin = xmin_horizon;
+		SpinLockRelease(&slot->mutex);
+
+		ReplicationSlotsComputeRequiredXmin(true);
+
+		LWLockRelease(ProcArrayLock);
+	}
+
 	ctx = StartupDecodingContext(output_plugin_options,
-								 start_lsn, InvalidTransactionId, false,
+								 start_lsn, xmin_horizon, need_full_snapshot,
 								 fast_forward, xl_routine, prepare_write,
 								 do_write, update_progress);
 
diff --git a/src/backend/replication/logical/logicalfuncs.c b/src/backend/replication/logical/logicalfuncs.c
index 55a24c02c9..85c5cdb633 100644
--- a/src/backend/replication/logical/logicalfuncs.c
+++ b/src/backend/replication/logical/logicalfuncs.c
@@ -208,6 +208,7 @@ pg_logical_slot_get_changes_guts(FunctionCallInfo fcinfo, bool confirm, bool bin
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									options,
 									false,
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y
index 0c874e33cf..957a5cc653 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -65,6 +65,7 @@ Node *replication_parse_result;
 %token K_CREATE_REPLICATION_SLOT
 %token K_DROP_REPLICATION_SLOT
 %token K_TIMELINE_HISTORY
+%token K_CREATE_REPLICATION_SNAPSHOT
 %token K_WAIT
 %token K_TIMELINE
 %token K_PHYSICAL
@@ -80,7 +81,7 @@ Node *replication_parse_result;
 %type <node>	command
 %type <node>	base_backup start_replication start_logical_replication
 				create_replication_slot drop_replication_slot identify_system
-				read_replication_slot timeline_history show
+				read_replication_slot timeline_history show create_replication_snapshot
 %type <list>	generic_option_list
 %type <defelt>	generic_option
 %type <uintval>	opt_timeline
@@ -114,6 +115,7 @@ command:
 			| read_replication_slot
 			| timeline_history
 			| show
+			| create_replication_snapshot
 			;
 
 /*
@@ -307,6 +309,19 @@ timeline_history:
 				}
 			;
 
+/*
+ * CREATE_REPLICATION_SNAPSHOT %s options
+ */
+create_replication_snapshot:
+			K_CREATE_REPLICATION_SNAPSHOT var_name plugin_options
+				{
+					CreateReplicationSnapshotCmd *n = makeNode(CreateReplicationSnapshotCmd);
+					n->slotname = $2;
+					n->options = $3;
+					$$ = (Node *) n;
+				}
+			;
+
 opt_physical:
 			K_PHYSICAL
 			| /* EMPTY */
@@ -400,6 +415,7 @@ ident_or_keyword:
 			| K_CREATE_REPLICATION_SLOT	{ $$ = "create_replication_slot"; }
 			| K_DROP_REPLICATION_SLOT		{ $$ = "drop_replication_slot"; }
 			| K_TIMELINE_HISTORY			{ $$ = "timeline_history"; }
+			| K_CREATE_REPLICATION_SNAPSHOT	{ $$ = "create_replication_snapshot"; }
 			| K_WAIT						{ $$ = "wait"; }
 			| K_TIMELINE					{ $$ = "timeline"; }
 			| K_PHYSICAL					{ $$ = "physical"; }
diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l
index cb467ca46f..5ba2e9c54b 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -126,6 +126,7 @@ START_REPLICATION	{ return K_START_REPLICATION; }
 CREATE_REPLICATION_SLOT		{ return K_CREATE_REPLICATION_SLOT; }
 DROP_REPLICATION_SLOT		{ return K_DROP_REPLICATION_SLOT; }
 TIMELINE_HISTORY	{ return K_TIMELINE_HISTORY; }
+CREATE_REPLICATION_SNAPSHOT	{ return K_CREATE_REPLICATION_SNAPSHOT; }
 PHYSICAL			{ return K_PHYSICAL; }
 RESERVE_WAL			{ return K_RESERVE_WAL; }
 LOGICAL				{ return K_LOGICAL; }
@@ -303,6 +304,7 @@ replication_scanner_is_replication_command(void)
 		case K_DROP_REPLICATION_SLOT:
 		case K_READ_REPLICATION_SLOT:
 		case K_TIMELINE_HISTORY:
+		case K_CREATE_REPLICATION_SNAPSHOT:
 		case K_SHOW:
 			/* Yes; push back the first token so we can parse later. */
 			repl_pushed_back_token = first_token;
diff --git a/src/backend/replication/slotfuncs.c b/src/backend/replication/slotfuncs.c
index 6035cf4816..c5b2d5b61f 100644
--- a/src/backend/replication/slotfuncs.c
+++ b/src/backend/replication/slotfuncs.c
@@ -486,6 +486,7 @@ pg_logical_replication_slot_advance(XLogRecPtr moveto)
 		ctx = CreateDecodingContext(InvalidXLogRecPtr,
 									NIL,
 									true,	/* fast_forward */
+									false,
 									XL_ROUTINE(.page_read = read_local_xlog_page,
 											   .segment_open = wal_segment_open,
 											   .segment_close = wal_segment_close),
diff --git a/src/backend/replication/walsender.c b/src/backend/replication/walsender.c
index 06adcf6f01..75ef245380 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -238,6 +238,7 @@ static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd);
 static void DropReplicationSlot(DropReplicationSlotCmd *cmd);
 static void StartReplication(StartReplicationCmd *cmd);
 static void StartLogicalReplication(StartReplicationCmd *cmd);
+static void CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd);
 static void ProcessStandbyMessage(void);
 static void ProcessStandbyReplyMessage(void);
 static void ProcessStandbyHSFeedbackMessage(void);
@@ -1283,7 +1284,7 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	 * are reported early.
 	 */
 	logical_decoding_ctx =
-		CreateDecodingContext(cmd->startpoint, cmd->options, false,
+		CreateDecodingContext(cmd->startpoint, cmd->options, false, false,
 							  XL_ROUTINE(.page_read = logical_read_xlog_page,
 										 .segment_open = WalSndSegmentOpen,
 										 .segment_close = wal_segment_close),
@@ -1335,6 +1336,98 @@ StartLogicalReplication(StartReplicationCmd *cmd)
 	EndCommand(&qc, DestRemote, false);
 }
 
+/*
+ * Create a snapshot from an existing replication slot.
+ */
+static void
+CreateReplicationSnapshot(CreateReplicationSnapshotCmd *cmd)
+{
+	Snapshot	snap;
+	LogicalDecodingContext *ctx;
+	char		xloc[MAXFNAMELEN];
+	DestReceiver *dest;
+	TupOutputState *tstate;
+	TupleDesc	tupdesc;
+	Datum		values[1];
+	bool		nulls[1] = {0};
+
+	Assert(!MyReplicationSlot);
+
+	CheckLogicalDecodingRequirements();
+
+	if (!IsTransactionBlock())
+		ereport(ERROR,
+				(errmsg("%s must be called inside a transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (XactIsoLevel != XACT_REPEATABLE_READ)
+		ereport(ERROR,
+				(errmsg("%s must be called in REPEATABLE READ isolation mode transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (!XactReadOnly)
+		ereport(ERROR,
+				(errmsg("%s must be called in a read only transaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (FirstSnapshotSet)
+		ereport(ERROR,
+				(errmsg("%s must be called before any query",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	if (IsSubTransaction())
+		ereport(ERROR,
+				(errmsg("%s must not be called in a subtransaction",
+						"CREATE_REPLICATION_SNAPSHOT ...")));
+
+	ReplicationSlotAcquire(cmd->slotname, false);
+
+	ctx = CreateDecodingContext(MyReplicationSlot->data.restart_lsn,
+								cmd->options,
+								false,
+								true,
+								XL_ROUTINE(.page_read = logical_read_xlog_page,
+										   .segment_open = WalSndSegmentOpen,
+										   .segment_close = wal_segment_close),
+								WalSndPrepareWrite, WalSndWriteData,
+								WalSndUpdateProgress);
+
+	/*
+	 * Signal that we don't need the timeout mechanism. We're just creating
+	 * the snapshot with the replication slot and don't yet accept feedback
+	 * messages or send keepalives. As we possibly need to wait for further
+	 * WAL the walsender would otherwise possibly be killed too soon.
+	 */
+	last_reply_timestamp = 0;
+
+	/* build initial snapshot, might take a while */
+	DecodingContextFindStartpoint(ctx);
+
+	snap = SnapBuildInitialSnapshot(ctx->snapshot_builder);
+	RestoreTransactionSnapshot(snap, MyProc);
+
+	/* Don't need the decoding context anymore */
+	FreeDecodingContext(ctx);
+
+	/* Create a tuple to send consistent WAL location */
+	snprintf(xloc, sizeof(xloc), "%X/%X",
+			 LSN_FORMAT_ARGS(MyReplicationSlot->data.confirmed_flush));
+
+	dest = CreateDestReceiver(DestRemoteSimple);
+	tupdesc = CreateTemplateTupleDesc(1);
+	TupleDescInitBuiltinEntry(tupdesc, (AttrNumber) 1, "consistent_point",
+							  TEXTOID, -1, 0);
+	tstate = begin_tup_output_tupdesc(dest, tupdesc, &TTSOpsVirtual);
+
+	/* consistent wal location */
+	values[0] = CStringGetTextDatum(xloc);
+
+	do_tup_output(tstate, values, nulls);
+	end_tup_output(tstate);
+
+	ReplicationSlotRelease();
+}
+
 /*
  * LogicalDecodingContext 'prepare_write' callback.
  *
@@ -1869,6 +1962,15 @@ exec_replication_command(const char *cmd_string)
 			}
 			break;
 
+		case T_CreateReplicationSnapshotCmd:
+			{
+				cmdtag = "CREATE_REPLICATION_SNAPSHOT";
+				set_ps_display(cmdtag);
+				CreateReplicationSnapshot((CreateReplicationSnapshotCmd *) cmd_node);
+				EndReplicationCommand(cmdtag);
+				break;
+			}
+
 		default:
 			elog(ERROR, "unrecognized replication command node tag: %u",
 				 cmd_node->type);
diff --git a/src/include/nodes/replnodes.h b/src/include/nodes/replnodes.h
index 4321ba8f86..154ab74f33 100644
--- a/src/include/nodes/replnodes.h
+++ b/src/include/nodes/replnodes.h
@@ -108,4 +108,15 @@ typedef struct TimeLineHistoryCmd
 	TimeLineID	timeline;
 } TimeLineHistoryCmd;
 
+/* ----------------------
+ *		CREATE_REPLICATION_SNAPSHOT command
+ * ----------------------
+ */
+typedef struct CreateReplicationSnapshotCmd
+{
+	NodeTag		type;
+	char	   *slotname;
+	List	   *options;
+} CreateReplicationSnapshotCmd;
+
 #endif							/* REPLNODES_H */
diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h
index 5f49554ea0..6535786a0e 100644
--- a/src/include/replication/logical.h
+++ b/src/include/replication/logical.h
@@ -125,6 +125,7 @@ extern LogicalDecodingContext *CreateInitDecodingContext(const char *plugin,
 extern LogicalDecodingContext *CreateDecodingContext(XLogRecPtr start_lsn,
 													 List *output_plugin_options,
 													 bool fast_forward,
+													 bool need_full_snapshot,
 													 XLogReaderRoutine *xl_routine,
 													 LogicalOutputPluginWriterPrepareWrite prepare_write,
 													 LogicalOutputPluginWriterWrite do_write,
diff --git a/src/include/replication/walreceiver.h b/src/include/replication/walreceiver.h
index 281626fa6f..f01e5f23a7 100644
--- a/src/include/replication/walreceiver.h
+++ b/src/include/replication/walreceiver.h
@@ -386,6 +386,16 @@ typedef WalRcvExecResult *(*walrcv_exec_fn) (WalReceiverConn *conn,
  */
 typedef void (*walrcv_disconnect_fn) (WalReceiverConn *conn);
 
+/*
+ * walrcv_slot_snapshot_fn
+ *
+ * Create a snapshot by an existing replication slot
+ */
+typedef void (*walrcv_slot_snapshot_fn) (WalReceiverConn *conn,
+										 char *slotname,
+										 const WalRcvStreamOptions *options,
+										 XLogRecPtr *lsn);
+
 typedef struct WalReceiverFunctionsType
 {
 	walrcv_connect_fn walrcv_connect;
@@ -403,6 +413,7 @@ typedef struct WalReceiverFunctionsType
 	walrcv_get_backend_pid_fn walrcv_get_backend_pid;
 	walrcv_exec_fn walrcv_exec;
 	walrcv_disconnect_fn walrcv_disconnect;
+	walrcv_slot_snapshot_fn walrcv_slot_snapshot;
 } WalReceiverFunctionsType;
 
 extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
@@ -437,6 +448,8 @@ extern PGDLLIMPORT WalReceiverFunctionsType *WalReceiverFunctions;
 	WalReceiverFunctions->walrcv_exec(conn, exec, nRetTypes, retTypes)
 #define walrcv_disconnect(conn) \
 	WalReceiverFunctions->walrcv_disconnect(conn)
+#define walrcv_slot_snapshot(conn, slotname, options, lsn) \
+	WalReceiverFunctions->walrcv_slot_snapshot(conn, slotname, options, lsn)
 
 static inline void
 walrcv_clear_result(WalRcvExecResult *walres)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index e941fb6c82..cc023b10fa 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2348,6 +2348,7 @@ ReplicationSlotInvalidationCause
 ReplicationSlotOnDisk
 ReplicationSlotPersistency
 ReplicationSlotPersistentData
+CreateReplicationSnapshotCmd
 ReplicationState
 ReplicationStateCtl
 ReplicationStateOnDisk
@@ -3855,6 +3856,7 @@ walrcv_receive_fn
 walrcv_send_fn
 walrcv_server_version_fn
 walrcv_startstreaming_fn
+walrcv_slot_snapshot_fn
 wchar2mb_with_len_converter
 wchar_t
 win32_deadchild_waitinfo
-- 
2.27.0

From ba307b7714a7da7d89bae9a0670e9ab2aed38632 Mon Sep 17 00:00:00 2001
From: Melih Mutlu <m.melihmutlu@gmail.com>
Date: Thu, 2 Jun 2022 17:39:37 +0300
Subject: [PATCH v16 5/5] Reuse Replication Slot and Origin in Tablesync

This commit allows reusing replication slots and origins during tablesync.

Earlier, a tablesync worker was creating a new replication slot and origin
each time it syncs a new table. With this patch, replication
slots/origins can be reusable for tablesync.

This reduces the overhead of creating/dropping replication slots and origins
and improves tablesync speed significantly especially for empty or small tables.

If the state of the current table is INIT or DATASYNC, tablesync worker needs a
repliation slot/origin. If the worker has not created slot and origin in
its previous runs, it will create those first. Otherwise the worker reuses
slot and origin created by the same worker in previous iterations earlier.
Tables in FINISHEDCOPY are expected to have a replication slot and origin.
Slot and origin names for such tables are persisted in
pg_subscription_rel catalog. Tablesync worker can fetch them and proceed
with existing slot and origin of FINISHEDCOPY tables and does not need to
create new ones.

Discussion: http://postgr.es/m/CAGPVpCTq=rUDd4JUdaRc1XUWf4BrH2gdSNf3rtOMUGj9rPpfzQ@mail.gmail.com
---
 doc/src/sgml/catalogs.sgml                    |  31 ++
 src/backend/catalog/pg_subscription.c         | 247 +++++++++++-
 src/backend/commands/subscriptioncmds.c       | 229 +++++++----
 .../replication/logical/applyparallelworker.c |   3 +-
 src/backend/replication/logical/launcher.c    |   8 +-
 src/backend/replication/logical/tablesync.c   | 373 +++++++++++++-----
 src/backend/replication/logical/worker.c      |  57 ++-
 src/include/catalog/pg_subscription.h         |   6 +
 src/include/catalog/pg_subscription_rel.h     |  14 +-
 src/include/replication/slot.h                |   3 +-
 src/include/replication/worker_internal.h     |  23 +-
 11 files changed, 803 insertions(+), 191 deletions(-)

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index 852cb30ae1..60718ab587 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -8030,6 +8030,19 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        origin.
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>sublastusedid</structfield> <type>int8</type>
+      </para>
+      <para>
+      The last used ID for tablesync workers. It acts as an unique identifier
+      for replication slots which are created by tablesync workers.
+      The last used ID needs to be persisted to make logical replication safely
+      proceed after any interruption. If sublastusedid is 0, then no table has
+      been synced yet.
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
@@ -8114,6 +8127,24 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
        otherwise null
       </para></entry>
      </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srrelslotname</structfield> <type>name</type>
+      </para>
+      <para>
+       Replication slot name that is used for synchronization of relation
+      </para></entry>
+     </row>
+
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+       <structfield>srreloriginname</structfield> <type>name</type>
+      </para>
+      <para>
+       Origin name that is used for tracking synchronization of relation
+      </para></entry>
+     </row>
     </tbody>
    </tgroup>
   </table>
diff --git a/src/backend/catalog/pg_subscription.c b/src/backend/catalog/pg_subscription.c
index d07f88ce28..152fdaa310 100644
--- a/src/backend/catalog/pg_subscription.c
+++ b/src/backend/catalog/pg_subscription.c
@@ -108,6 +108,14 @@ GetSubscription(Oid subid, bool missing_ok)
 								   Anum_pg_subscription_suborigin);
 	sub->origin = TextDatumGetCString(datum);
 
+	/* Get last used id */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID,
+							tup,
+							Anum_pg_subscription_sublastusedid,
+							&isnull);
+	Assert(!isnull);
+	sub->lastusedid = DatumGetInt64(datum);
+
 	ReleaseSysCache(tup);
 
 	return sub;
@@ -199,6 +207,44 @@ DisableSubscription(Oid subid)
 	table_close(rel, NoLock);
 }
 
+/*
+ * Update the last used replication slot ID for the given subscription.
+ */
+void
+UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid)
+{
+	Relation	rel;
+	bool		nulls[Natts_pg_subscription];
+	bool		replaces[Natts_pg_subscription];
+	Datum		values[Natts_pg_subscription];
+	HeapTuple	tup;
+
+	/* Look up the subscription in the catalog */
+	rel = table_open(SubscriptionRelationId, RowExclusiveLock);
+	tup = SearchSysCacheCopy1(SUBSCRIPTIONOID, ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "cache lookup failed for subscription %u", subid);
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	/* Form a new tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	replaces[Anum_pg_subscription_sublastusedid - 1] = true;
+	values[Anum_pg_subscription_sublastusedid- 1] = Int64GetDatum(lastusedid);
+
+	/* Update the catalog */
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+	heap_freetuple(tup);
+
+	table_close(rel, NoLock);
+}
+
 /*
  * Convert text array to list of strings.
  *
@@ -228,7 +274,7 @@ textarray_to_stringlist(ArrayType *textarray)
  */
 void
 AddSubscriptionRelState(Oid subid, Oid relid, char state,
-						XLogRecPtr sublsn)
+						XLogRecPtr sublsn, char *relslotname, char *reloriginname)
 {
 	Relation	rel;
 	HeapTuple	tup;
@@ -257,6 +303,16 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
 	else
 		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -269,6 +325,60 @@ AddSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Internal function to modify columns for relation state update
+ */
+static void
+UpdateSubscriptionRelState_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char state,
+									XLogRecPtr sublsn)
+{
+	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
+	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+
+	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	if (sublsn != InvalidXLogRecPtr)
+		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
+	else
+		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication slot update
+ */
+static void
+UpdateSubscriptionRelReplicationSlot_internal(Datum *values,
+											bool *nulls,
+											bool *replaces,
+											char *relslotname)
+{
+	replaces[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+	if (relslotname)
+		values[Anum_pg_subscription_rel_srrelslotname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(relslotname));
+	else
+		nulls[Anum_pg_subscription_rel_srrelslotname - 1] = true;
+}
+
+/*
+ * Internal function to modify columns for replication origin update
+ */
+static void
+UpdateSubscriptionRelOrigin_internal(Datum *values,
+									bool *nulls,
+									bool *replaces,
+									char *reloriginname)
+{
+	replaces[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+	if (reloriginname)
+		values[Anum_pg_subscription_rel_srreloriginname - 1] =
+			DirectFunctionCall1(namein, CStringGetDatum(reloriginname));
+	else
+		nulls[Anum_pg_subscription_rel_srreloriginname - 1] = true;
+}
+
 /*
  * Update the state of a subscription table.
  */
@@ -299,14 +409,56 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	memset(nulls, false, sizeof(nulls));
 	memset(replaces, false, sizeof(replaces));
 
-	replaces[Anum_pg_subscription_rel_srsubstate - 1] = true;
-	values[Anum_pg_subscription_rel_srsubstate - 1] = CharGetDatum(state);
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
 
-	replaces[Anum_pg_subscription_rel_srsublsn - 1] = true;
-	if (sublsn != InvalidXLogRecPtr)
-		values[Anum_pg_subscription_rel_srsublsn - 1] = LSNGetDatum(sublsn);
-	else
-		nulls[Anum_pg_subscription_rel_srsublsn - 1] = true;
+	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
+							replaces);
+
+	/* Update the catalog. */
+	CatalogTupleUpdate(rel, &tup->t_self, tup);
+
+	/* Cleanup. */
+	table_close(rel, NoLock);
+}
+
+/*
+ * Update replication slot name, origin name and state of
+ * a subscription table in one transaction.
+ */
+void
+UpdateSubscriptionRel(Oid subid,
+					  Oid relid,
+					  char state,
+					  XLogRecPtr sublsn,
+					  char *relslotname,
+					  char *reloriginname)
+{
+	Relation	rel;
+	HeapTuple	tup;
+	bool		nulls[Natts_pg_subscription_rel];
+	Datum		values[Natts_pg_subscription_rel];
+	bool		replaces[Natts_pg_subscription_rel];
+
+	LockSharedObject(SubscriptionRelationId, subid, 0, AccessShareLock);
+
+	rel = table_open(SubscriptionRelRelationId, RowExclusiveLock);
+
+	/* Try finding existing mapping. */
+	tup = SearchSysCacheCopy2(SUBSCRIPTIONRELMAP,
+							  ObjectIdGetDatum(relid),
+							  ObjectIdGetDatum(subid));
+	if (!HeapTupleIsValid(tup))
+		elog(ERROR, "subscription table %u in subscription %u does not exist",
+			 relid, subid);
+
+	/* Update the tuple. */
+	memset(values, 0, sizeof(values));
+	memset(nulls, false, sizeof(nulls));
+	memset(replaces, false, sizeof(replaces));
+
+	UpdateSubscriptionRelState_internal(values, nulls, replaces, state, sublsn);
+	UpdateSubscriptionRelReplicationSlot_internal(values, nulls, replaces, relslotname);
+	UpdateSubscriptionRelOrigin_internal(values, nulls, replaces, reloriginname);
 
 	tup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls,
 							replaces);
@@ -318,6 +470,85 @@ UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 	table_close(rel, NoLock);
 }
 
+/*
+ * Get origin name of subscription table.
+ *
+ * reloriginname's value has the replication origin name if the origin exists.
+ */
+void
+GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*originname;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srreloriginname, isnull);
+	if (!*isnull)
+	{
+		originname = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(reloriginname, originname, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
+/*
+ * Get replication slot name of subscription table.
+ *
+ * slotname's value has the replication slot name if the subscription has any.
+ */
+void
+GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname)
+{
+	HeapTuple	tup;
+	Relation	rel;
+	Datum 		d;
+	char		*relrepslot;
+	bool		isnull;
+
+	rel = table_open(SubscriptionRelRelationId, AccessShareLock);
+
+	/* Try finding the mapping. */
+	tup = SearchSysCache2(SUBSCRIPTIONRELMAP,
+						  ObjectIdGetDatum(relid),
+						  ObjectIdGetDatum(subid));
+
+	if (!HeapTupleIsValid(tup))
+	{
+		table_close(rel, AccessShareLock);
+	}
+
+	d = SysCacheGetAttr(SUBSCRIPTIONRELMAP, tup,
+						Anum_pg_subscription_rel_srrelslotname, &isnull);
+	if (!isnull)
+	{
+		relrepslot = DatumGetCString(DirectFunctionCall1(nameout, d));
+		memcpy(slotname, relrepslot, NAMEDATALEN);
+	}
+
+	/* Cleanup */
+	ReleaseSysCache(tup);
+
+	table_close(rel, AccessShareLock);
+}
+
 /*
  * Get state of subscription table.
  *
diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c
index d4e798baeb..4f5b3c572c 100644
--- a/src/backend/commands/subscriptioncmds.c
+++ b/src/backend/commands/subscriptioncmds.c
@@ -710,6 +710,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 		publicationListToArray(publications);
 	values[Anum_pg_subscription_suborigin - 1] =
 		CStringGetTextDatum(opts.origin);
+	values[Anum_pg_subscription_sublastusedid - 1] = Int64GetDatum(0);
 
 	tup = heap_form_tuple(RelationGetDescr(rel), values, nulls);
 
@@ -773,7 +774,7 @@ CreateSubscription(ParseState *pstate, CreateSubscriptionStmt *stmt,
 										 rv->schemaname, rv->relname);
 
 				AddSubscriptionRelState(subid, relid, table_state,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 			}
 
 			/*
@@ -864,6 +865,8 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 	SubRemoveRels *sub_remove_rels;
 	WalReceiverConn *wrconn;
 	bool		must_use_password;
+	List	   *sub_remove_slots = NIL;
+	LogicalRepWorker *worker;
 
 	/* Load the library providing us libpq calls. */
 	load_file("libpqwalreceiver", false);
@@ -943,7 +946,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				AddSubscriptionRelState(sub->oid, relid,
 										copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY,
-										InvalidXLogRecPtr);
+										InvalidXLogRecPtr, NULL, NULL);
 				ereport(DEBUG1,
 						(errmsg_internal("table \"%s.%s\" added to subscription \"%s\"",
 										 rv->schemaname, rv->relname, sub->name)));
@@ -967,6 +970,7 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 			{
 				char		state;
 				XLogRecPtr	statelsn;
+				char		slotname[NAMEDATALEN] = {0};
 
 				/*
 				 * Lock pg_subscription_rel with AccessExclusiveLock to
@@ -993,13 +997,36 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 
 				RemoveSubscriptionRel(sub->oid, relid);
 
-				logicalrep_worker_stop(sub->oid, relid);
+				/*
+				 * Find the logical replication sync worker. If exists, store
+				 * the slot number for dropping associated replication slots
+				 * later.
+				 */
+				LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+				worker = logicalrep_worker_find(sub->oid, relid, false);
+				if (worker)
+				{
+					logicalrep_worker_stop(sub->oid, relid);
+					sub_remove_slots = lappend(sub_remove_slots, &worker->slot_name);
+				}
+				else
+				{
+					/*
+					 * Sync of this relation might be failed in an earlier
+					 * attempt, but the replication slot might still exist.
+					 */
+					GetSubscriptionRelReplicationSlot(sub->oid, relid, slotname);
+					if (strlen(slotname) > 0)
+						sub_remove_slots = lappend(sub_remove_slots, slotname);
+				}
+				LWLockRelease(LogicalRepWorkerLock);
 
 				/*
 				 * For READY state, we would have already dropped the
 				 * tablesync origin.
 				 */
-				if (state != SUBREL_STATE_READY)
+				if (state != SUBREL_STATE_READY &&
+					state != SUBREL_STATE_SYNCDONE)
 				{
 					char		originname[NAMEDATALEN];
 
@@ -1027,31 +1054,24 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data,
 		}
 
 		/*
-		 * Drop the tablesync slots associated with removed tables. This has
-		 * to be at the end because otherwise if there is an error while doing
-		 * the database operations we won't be able to rollback dropped slots.
+		 * Drop the replication slots associated with tablesync workers for
+		 * removed tables. This has to be at the end because otherwise if
+		 * there is an error while doing the database operations we won't be
+		 * able to rollback dropped slots.
 		 */
-		for (off = 0; off < remove_rel_len; off++)
+		foreach(lc, sub_remove_slots)
 		{
-			if (sub_remove_rels[off].state != SUBREL_STATE_READY &&
-				sub_remove_rels[off].state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+			char		syncslotname[NAMEDATALEN] = {0};
 
-				/*
-				 * For READY/SYNCDONE states we know the tablesync slot has
-				 * already been dropped by the tablesync worker.
-				 *
-				 * For other states, there is no certainty, maybe the slot
-				 * does not exist yet. Also, if we fail after removing some of
-				 * the slots, next time, it will again try to drop already
-				 * dropped slots and fail. For these reasons, we allow
-				 * missing_ok = true for the drop.
-				 */
-				ReplicationSlotNameForTablesync(sub->oid, sub_remove_rels[off].relid,
-												syncslotname, sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			memcpy(syncslotname, lfirst(lc), sizeof(NAMEDATALEN));
+
+			/*
+			 * There is no certainty, maybe the slot does not exist yet. Also,
+			 * if we fail after removing some of the slots, next time, it will
+			 * again try to drop already dropped slots and fail. For these
+			 * reasons, we allow missing_ok = true for the drop.
+			 */
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 	}
 	PG_FINALLY();
@@ -1474,6 +1494,7 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	char	   *subname;
 	char	   *conninfo;
 	char	   *slotname;
+	int64		lastusedid;
 	List	   *subworkers;
 	ListCell   *lc;
 	char		originname[NAMEDATALEN];
@@ -1546,6 +1567,14 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	else
 		slotname = NULL;
 
+	/* Get the last used identifier by the subscription */
+	datum = SysCacheGetAttr(SUBSCRIPTIONOID, tup,
+							Anum_pg_subscription_sublastusedid, &isnull);
+	if (!isnull)
+		lastusedid = DatumGetInt64(datum);
+	else
+		lastusedid = 0;
+
 	/*
 	 * Since dropping a replication slot is not transactional, the replication
 	 * slot stays dropped even if the transaction rolls back.  So we cannot
@@ -1595,6 +1624,8 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	}
 	list_free(subworkers);
 
+	rstates = GetSubscriptionRelations(subid, true);
+
 	/*
 	 * Remove the no-longer-useful entry in the launcher's table of apply
 	 * worker start times.
@@ -1606,36 +1637,26 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	ApplyLauncherForgetWorkerStartTime(subid);
 
 	/*
-	 * Cleanup of tablesync replication origins.
-	 *
-	 * Any READY-state relations would already have dealt with clean-ups.
+	 * Cleanup of tablesync replication origins associated with the
+	 * subscription, if exists. Try to drop origins by creating all origin
+	 * names created for this subscription.
 	 *
 	 * Note that the state can't change because we have already stopped both
 	 * the apply and tablesync workers and they can't restart because of
 	 * exclusive lock on the subscription.
+	 *
+	 * XXX: This can be handled better instead of looping through all possible
 	 */
-	rstates = GetSubscriptionRelations(subid, true);
-	foreach(lc, rstates)
+	for (int64 i = 1; i <= lastusedid; i++)
 	{
-		SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-		Oid			relid = rstate->relid;
-
-		/* Only cleanup resources of tablesync workers */
-		if (!OidIsValid(relid))
-			continue;
+		char		originname_to_drop[NAMEDATALEN] = {0};
 
-		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * It is possible that the origin is not yet created for tablesync
-		 * worker so passing missing_ok = true. This can happen for the states
-		 * before SUBREL_STATE_FINISHEDCOPY.
-		 */
-		ReplicationOriginNameForLogicalRep(subid, relid, originname,
-										   sizeof(originname));
-		replorigin_drop_by_name(originname, true, false);
+		snprintf(originname_to_drop, sizeof(originname_to_drop), "pg_%u_%lld", subid, (long long) i);
+		/* missing_ok = true, since the origin might be already dropped. */
+		replorigin_drop_by_name(originname_to_drop, true, false);
 	}
 
+
 	/* Clean up dependencies */
 	deleteSharedDependencyRecordsFor(SubscriptionRelationId, subid, 0);
 
@@ -1694,39 +1715,17 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 
 	PG_TRY();
 	{
-		foreach(lc, rstates)
-		{
-			SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc);
-			Oid			relid = rstate->relid;
+		List	   *slots = NULL;
 
-			/* Only cleanup resources of tablesync workers */
-			if (!OidIsValid(relid))
-				continue;
 
-			/*
-			 * Drop the tablesync slots associated with removed tables.
-			 *
-			 * For SYNCDONE/READY states, the tablesync slot is known to have
-			 * already been dropped by the tablesync worker.
-			 *
-			 * For other states, there is no certainty, maybe the slot does
-			 * not exist yet. Also, if we fail after removing some of the
-			 * slots, next time, it will again try to drop already dropped
-			 * slots and fail. For these reasons, we allow missing_ok = true
-			 * for the drop.
-			 */
-			if (rstate->state != SUBREL_STATE_SYNCDONE)
-			{
-				char		syncslotname[NAMEDATALEN] = {0};
+		slots = GetReplicationSlotNamesBySubId(wrconn, subid, true);
+		foreach(lc, slots)
+		{
+			char	   *syncslotname = (char *) lfirst(lc);
 
-				ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-												sizeof(syncslotname));
-				ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
-			}
+			ReplicationSlotDropAtPubNode(wrconn, syncslotname, true);
 		}
 
-		list_free(rstates);
-
 		/*
 		 * If there is a slot associated with the subscription, then drop the
 		 * replication slot at the publisher.
@@ -1743,6 +1742,71 @@ DropSubscription(DropSubscriptionStmt *stmt, bool isTopLevel)
 	table_close(rel, NoLock);
 }
 
+/*
+ * GetReplicationSlotNamesBySubId
+ *
+ * Get the replication slot names associated with the subscription.
+ */
+List *
+GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok)
+{
+	StringInfoData cmd;
+	TupleTableSlot *slot;
+	Oid			tableRow[1] = {NAMEOID};
+	List	   *tablelist = NIL;
+
+	Assert(wrconn);
+
+	load_file("libpqwalreceiver", false);
+
+	initStringInfo(&cmd);
+	appendStringInfo(&cmd, "SELECT slot_name"
+					 " FROM pg_replication_slots"
+					 " WHERE slot_name LIKE 'pg_%i_sync_%%';",
+					 subid);
+	PG_TRY();
+	{
+		WalRcvExecResult *res;
+
+		res = walrcv_exec(wrconn, cmd.data, 1, tableRow);
+
+		if (res->status != WALRCV_OK_TUPLES)
+		{
+			ereport(ERROR,
+					errmsg("could not receive list of slots associated with the subscription %u, error: %s",
+					subid, res->err));
+		}
+
+		/* Process tables. */
+		slot = MakeSingleTupleTableSlot(res->tupledesc, &TTSOpsMinimalTuple);
+		while (tuplestore_gettupleslot(res->tuplestore, true, false, slot))
+		{
+			char	   *repslotname;
+			char	   *slotattr;
+			bool		isnull;
+
+			slotattr = NameStr(*DatumGetName(slot_getattr(slot, 1, &isnull)));
+			Assert(!isnull);
+
+			repslotname = palloc(sizeof(char) * strlen(slotattr) + 1);
+			memcpy(repslotname, slotattr, sizeof(char) * strlen(slotattr));
+			repslotname[strlen(slotattr)] = '\0';
+			tablelist = lappend(tablelist, repslotname);
+
+			ExecClearTuple(slot);
+		}
+		ExecDropSingleTupleTableSlot(slot);
+
+		walrcv_clear_result(res);
+	}
+	PG_FINALLY();
+	{
+		pfree(cmd.data);
+	}
+	PG_END_TRY();
+		return tablelist;
+}
+
 /*
  * Drop the replication slot at the publisher node using the replication
  * connection.
@@ -2155,6 +2219,7 @@ static void
 ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 {
 	ListCell   *lc;
+	LogicalRepWorker *worker;
 
 	foreach(lc, rstates)
 	{
@@ -2165,18 +2230,20 @@ ReportSlotConnectionError(List *rstates, Oid subid, char *slotname, char *err)
 		if (!OidIsValid(relid))
 			continue;
 
+		/* Check if there is a sync worker for the relation */
+		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
+		worker = logicalrep_worker_find(subid, relid, false);
+		LWLockRelease(LogicalRepWorkerLock);
+
 		/*
 		 * Caller needs to ensure that relstate doesn't change underneath us.
 		 * See DropSubscription where we get the relstates.
 		 */
-		if (rstate->state != SUBREL_STATE_SYNCDONE)
+		if (worker &&
+			rstate->state != SUBREL_STATE_SYNCDONE)
 		{
-			char		syncslotname[NAMEDATALEN] = {0};
-
-			ReplicationSlotNameForTablesync(subid, relid, syncslotname,
-											sizeof(syncslotname));
 			elog(WARNING, "could not drop tablesync replication slot \"%s\"",
-				 syncslotname);
+				 worker->slot_name);
 		}
 	}
 
diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index 1d4e83c4c1..d2c70dffbc 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -440,7 +440,8 @@ pa_launch_parallel_worker(void)
 										MySubscription->name,
 										MyLogicalRepWorker->userid,
 										InvalidOid,
-										dsm_segment_handle(winfo->dsm_seg));
+										dsm_segment_handle(winfo->dsm_seg),
+										InvalidRepSlotId);
 
 	if (launched)
 	{
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index 72e5ef8a78..c84193319c 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -304,7 +304,7 @@ logicalrep_workers_find(Oid subid, bool only_running)
  */
 bool
 logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname, Oid userid,
-						 Oid relid, dsm_handle subworker_dsm)
+						 Oid relid, dsm_handle subworker_dsm, int64 slotid)
 {
 	BackgroundWorker bgw;
 	BackgroundWorkerHandle *bgw_handle;
@@ -430,6 +430,9 @@ retry:
 	worker->launch_time = now;
 	worker->in_use = true;
 	worker->generation++;
+	worker->created_slot = false;
+	worker->rep_slot_id = slotid;
+	worker->slot_name = (char *) palloc(NAMEDATALEN);
 	worker->proc = NULL;
 	worker->dbid = dbid;
 	worker->userid = userid;
@@ -1183,7 +1186,8 @@ ApplyLauncherMain(Datum main_arg)
 				ApplyLauncherSetWorkerStartTime(sub->oid, now);
 				logicalrep_worker_launch(sub->dbid, sub->oid, sub->name,
 										 sub->owner, InvalidOid,
-										 DSM_HANDLE_INVALID);
+										 DSM_HANDLE_INVALID,
+										 InvalidRepSlotId);
 			}
 			else
 			{
diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c
index f042d9ae00..830fa4696c 100644
--- a/src/backend/replication/logical/tablesync.c
+++ b/src/backend/replication/logical/tablesync.c
@@ -328,40 +328,29 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 								   MyLogicalRepWorker->relid,
 								   MyLogicalRepWorker->relstate,
 								   MyLogicalRepWorker->relstate_lsn);
+		CommitTransactionCommand();
 
 		/*
-		 * End streaming so that LogRepWorkerWalRcvConn can be used to drop
-		 * the slot.
-		 */
-		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
-
-		/*
-		 * Cleanup the tablesync slot.
+		 * Cleanup the tablesync slot. If the slot name used by this worker is
+		 * different from the default slot name for the worker, this means the
+		 * current table had started to being synchronized by another worker
+		 * and replication slot. And this worker is reusing a replication slot
+		 * from a previous attempt. We do not need that replication slot
+		 * anymore.
 		 *
 		 * This has to be done after updating the state because otherwise if
 		 * there is an error while doing the database operations we won't be
 		 * able to rollback dropped slot.
 		 */
 		ReplicationSlotNameForTablesync(MyLogicalRepWorker->subid,
-										MyLogicalRepWorker->relid,
+										MyLogicalRepWorker->rep_slot_id,
 										syncslotname,
 										sizeof(syncslotname));
 
 		/*
-		 * It is important to give an error if we are unable to drop the slot,
-		 * otherwise, it won't be dropped till the corresponding subscription
-		 * is dropped. So passing missing_ok = false.
-		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, syncslotname, false);
-
-		CommitTransactionCommand();
-		pgstat_report_stat(false);
-
-		/*
-		 * Start a new transaction to clean up the tablesync origin tracking.
-		 * This transaction will be ended within the finish_sync_worker().
-		 * Now, even, if we fail to remove this here, the apply worker will
-		 * ensure to clean it up afterward.
+		 * We are safe to drop the replication tracking origin after this
+		 * point. Now, even, if we fail to remove this here, the apply worker
+		 * will ensure to clean it up afterward.
 		 *
 		 * We need to do this after the table state is set to SYNCDONE.
 		 * Otherwise, if an error occurs while performing the database
@@ -370,32 +359,73 @@ process_syncing_tables_for_sync(XLogRecPtr current_lsn)
 		 * have been cleared before restart. So, the restarted worker will use
 		 * invalid replication progress state resulting in replay of
 		 * transactions that have already been applied.
+		 *
+		 * Firstly reset the origin session to remove the ownership of the
+		 * slot. This is needed to allow the origin to be dropped or reused
+		 * later.
 		 */
+		replorigin_session_reset();
+		replorigin_session_origin = InvalidRepOriginId;
+		replorigin_session_origin_lsn = InvalidXLogRecPtr;
+		replorigin_session_origin_timestamp = 0;
+
 		StartTransactionCommand();
+		if (MyLogicalRepWorker->slot_name && strcmp(syncslotname, MyLogicalRepWorker->slot_name) != 0)
+		{
+			/*
+			 * End streaming so that LogRepWorkerWalRcvConn can be used to
+			 * drop the slot.
+			 */
+			walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
 
-		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-										   MyLogicalRepWorker->relid,
-										   originname,
-										   sizeof(originname));
+			ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+											   MyLogicalRepWorker->relid,
+											   originname,
+											   sizeof(originname));
+
+			/*
+			 * Drop replication origin
+			 *
+			 * There is a chance that the user is concurrently performing refresh
+			 * for the subscription where we remove the table state and its origin
+			 * or the apply worker would have removed this origin. So passing
+			 * missing_ok = true.
+			 */
+			replorigin_drop_by_name(originname, true, false);
+		}
 
 		/*
-		 * Resetting the origin session removes the ownership of the slot.
-		 * This is needed to allow the origin to be dropped.
+		 * We are safe to remove persisted replication slot and origin data,
+		 * since it's already in SYNCDONE state. They will not be needed
+		 * anymore.
 		 */
-		replorigin_session_reset();
-		replorigin_session_origin = InvalidRepOriginId;
-		replorigin_session_origin_lsn = InvalidXLogRecPtr;
-		replorigin_session_origin_timestamp = 0;
+		UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+							  MyLogicalRepWorker->relid,
+							  MyLogicalRepWorker->relstate,
+							  MyLogicalRepWorker->relstate_lsn,
+							  NULL,
+							  NULL);
+		ereport(DEBUG2,
+				(errmsg("process_syncing_tables_for_sync: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL",
+						"NULL",
+						MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid,
+						MyLogicalRepWorker->subid)));
+		CommitTransactionCommand();
+		pgstat_report_stat(false);
 
 		/*
-		 * Drop the tablesync's origin tracking if exists.
-		 *
-		 * There is a chance that the user is concurrently performing refresh
-		 * for the subscription where we remove the table state and its origin
-		 * or the apply worker would have removed this origin. So passing
-		 * missing_ok = true.
+		 * This should return the default origin name for the worker. Even if
+		 * the worker used a different origin for this table, it should be
+		 * dropped and removed from the catalog so far.
 		 */
-		replorigin_drop_by_name(originname, true, false);
+		StartTransactionCommand();
+		ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
+										   MyLogicalRepWorker->relid,
+										   originname,
+										   sizeof(originname));
 
 		/* Sync worker has completed synchronization of the current table. */
 		MyLogicalRepWorker->is_sync_completed = true;
@@ -492,6 +522,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 			if (current_lsn >= rstate->lsn)
 			{
 				char		originname[NAMEDATALEN];
+				bool		is_origin_null = true;
 
 				rstate->state = SUBREL_STATE_READY;
 				rstate->lsn = current_lsn;
@@ -512,18 +543,31 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 				 * error while dropping we won't restart it to drop the
 				 * origin. So passing missing_ok = true.
 				 */
-				ReplicationOriginNameForLogicalRep(MyLogicalRepWorker->subid,
-												   rstate->relid,
-												   originname,
-												   sizeof(originname));
-				replorigin_drop_by_name(originname, true, false);
+				GetSubscriptionRelOrigin(MyLogicalRepWorker->subid,
+										 rstate->relid, originname,
+										 &is_origin_null);
+
+				if (!is_origin_null)
+				{
+					replorigin_drop_by_name(originname, true, false);
+				}
 
 				/*
 				 * Update the state to READY only after the origin cleanup.
 				 */
-				UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-										   rstate->relid, rstate->state,
-										   rstate->lsn);
+				UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+									  rstate->relid,
+									  rstate->state,
+									  rstate->lsn,
+									  NULL,
+									  NULL);
+				ereport(DEBUG2,
+					(errmsg("process_syncing_tables_for_apply: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+							"NULL", "NULL", rstate->state,
+							rstate->relid, MyLogicalRepWorker->subid)));
+
+				CommitTransactionCommand();
+				started_tx = false;
 			}
 		}
 		else
@@ -612,12 +656,25 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn)
 						TimestampDifferenceExceeds(hentry->last_start_time, now,
 												   wal_retrieve_retry_interval))
 					{
+						if (IsTransactionState())
+							CommitTransactionCommand();
+						StartTransactionCommand();
+						started_tx = true;
+
+						MySubscription->lastusedid++;
+						UpdateSubscriptionLastSlotId(MyLogicalRepWorker->subid,
+													 MySubscription->lastusedid);
+						ereport(DEBUG2,
+								(errmsg("process_syncing_tables_for_apply: incremented lastusedid to %lld for subscription %u",
+										(long long) MySubscription->lastusedid, MySubscription->oid)));
+
 						logicalrep_worker_launch(MyLogicalRepWorker->dbid,
 												 MySubscription->oid,
 												 MySubscription->name,
 												 MyLogicalRepWorker->userid,
 												 rstate->relid,
-												 DSM_HANDLE_INVALID);
+												 DSM_HANDLE_INVALID,
+												 MySubscription->lastusedid);
 						hentry->last_start_time = now;
 					}
 				}
@@ -1240,8 +1297,8 @@ copy_table(Relation rel)
  * The name must not exceed NAMEDATALEN - 1 because of remote node constraints
  * on slot name length. We append system_identifier to avoid slot_name
  * collision with subscriptions in other clusters. With the current scheme
- * pg_%u_sync_%u_UINT64_FORMAT (3 + 10 + 6 + 10 + 20 + '\0'), the maximum
- * length of slot_name will be 50.
+ * pg_%u_sync_%lu_UINT64_FORMAT (3 + 10 + 6 + 20 + 20 + '\0'), the maximum
+ * length of slot_name will be 45.
  *
  * The returned slot name is stored in the supplied buffer (syncslotname) with
  * the given size.
@@ -1252,11 +1309,11 @@ copy_table(Relation rel)
  * had changed.
  */
 void
-ReplicationSlotNameForTablesync(Oid suboid, Oid relid,
+ReplicationSlotNameForTablesync(Oid suboid, int64 slotid,
 								char *syncslotname, Size szslot)
 {
-	snprintf(syncslotname, szslot, "pg_%u_sync_%u_" UINT64_FORMAT, suboid,
-			 relid, GetSystemIdentifier());
+	snprintf(syncslotname, szslot, "pg_%u_sync_%lld_" UINT64_FORMAT, suboid,
+			(long long) slotid, GetSystemIdentifier());
 }
 
 /*
@@ -1282,6 +1339,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 	UserContext ucxt;
 	bool		must_use_password;
 	bool		run_as_owner;
+	char	   *prev_slotname;
 
 	/* Check the state of the table synchronization. */
 	StartTransactionCommand();
@@ -1316,7 +1374,7 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 	/* Calculate the name of the tablesync slot. */
 	slotname = (char *) palloc(NAMEDATALEN);
 	ReplicationSlotNameForTablesync(MySubscription->oid,
-									MyLogicalRepWorker->relid,
+									MyLogicalRepWorker->rep_slot_id,
 									slotname,
 									NAMEDATALEN);
 
@@ -1355,12 +1413,26 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC ||
 		   MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY);
 
+	/*
+	 * See if tablesync of the current relation has been started with another
+	 * replication slot.
+	 *
+	 * Read previous slot name from the catalog, if exists.
+	 */
+	prev_slotname = (char *) palloc(NAMEDATALEN);
+	StartTransactionCommand();
+	GetSubscriptionRelReplicationSlot(MyLogicalRepWorker->subid,
+									  MyLogicalRepWorker->relid,
+									  prev_slotname);
+
 	/* Assign the origin tracking record name. */
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   sizeof(originname));
 
+	CommitTransactionCommand();
+
 	if (MyLogicalRepWorker->relstate == SUBREL_STATE_DATASYNC)
 	{
 		/*
@@ -1374,10 +1446,53 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 		 * breakdown then it wouldn't have succeeded so trying it next time
 		 * seems like a better bet.
 		 */
-		ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, slotname, true);
+		if (strlen(prev_slotname) > 0)
+		{
+			ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, prev_slotname, true);
+
+			StartTransactionCommand();
+			/* Replication origin might still exist. Try to drop */
+			replorigin_drop_by_name(originname, true, false);
+
+			/*
+			 * Remove replication slot and origin name from the relation's
+			 * catalog record
+			 */
+			UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+								  MyLogicalRepWorker->relid,
+								  MyLogicalRepWorker->relstate,
+								  MyLogicalRepWorker->relstate_lsn,
+								  NULL,
+								  NULL);
+			CommitTransactionCommand();
+			ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+						"NULL", "NULL", MyLogicalRepWorker->relstate,
+						MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+		}
 	}
 	else if (MyLogicalRepWorker->relstate == SUBREL_STATE_FINISHEDCOPY)
 	{
+		/*
+		 * At this point, the table that is currently being synchronized
+		 * should have its replication slot name filled in the catalog. The
+		 * tablesync process was started with another sync worker and
+		 * replication slot. We need to continue using the same replication
+		 * slot in this worker too.
+		 */
+		if (strlen(prev_slotname) == 0)
+		{
+			elog(ERROR, "Replication slot could not be found for subscription %u, relation %u",
+				 MyLogicalRepWorker->subid,
+				 MyLogicalRepWorker->relid);
+		}
+
+		/*
+		 * Proceed with the correct replication slot. Use previously created
+		 * replication slot to sync this table.
+		 */
+		memcpy(slotname, prev_slotname, NAMEDATALEN);
+
 		/*
 		 * The COPY phase was previously done, but tablesync then crashed
 		 * before it was able to finish normally.
@@ -1397,7 +1512,9 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 
 		goto copy_table_done;
 	}
+	pfree(prev_slotname);
 
+	/* Preparing for table copy operation */
 	SpinLockAcquire(&MyLogicalRepWorker->relmutex);
 	MyLogicalRepWorker->relstate = SUBREL_STATE_DATASYNC;
 	MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr;
@@ -1405,11 +1522,31 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 
 	/* Update the state and make it visible to others. */
 	StartTransactionCommand();
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   MyLogicalRepWorker->relstate,
-							   MyLogicalRepWorker->relstate_lsn);
+
+	/*
+	 * Refresh the originname in case of having non-existing origin
+	 * from previous failed sync attempts.
+	 * If that's the case, it should be removed from the catalog so far.
+	 * Then, we can continue by reusing the origin created by the current
+	 * worker instead of .
+	 */
+	ReplicationOriginNameForLogicalRep(MySubscription->oid,
+									MyLogicalRepWorker->relid,
+									originname,
+									sizeof(originname));
+
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  MyLogicalRepWorker->relstate,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					slotname, originname, MyLogicalRepWorker->relstate,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
+
 	pgstat_report_stat(true);
 
 	StartTransactionCommand();
@@ -1437,48 +1574,96 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 						res->err)));
 	walrcv_clear_result(res);
 
+	originid = replorigin_by_name(originname, true);
+
 	/*
 	 * Create a new permanent logical decoding slot. This slot will be used
 	 * for the catchup phase after COPY is done, so tell it to use the
 	 * snapshot to make the final data consistent.
+	 *
+	 * Replication slot will only be created if either this is the first run
+	 * of the worker or we're not using a previous replication slot.
 	 */
-	walrcv_create_slot(LogRepWorkerWalRcvConn,
-					   slotname, false /* permanent */ , false /* two_phase */ ,
-					   CRS_USE_SNAPSHOT, origin_startpos);
-
-	/*
-	 * Setup replication origin tracking. The purpose of doing this before the
-	 * copy is to avoid doing the copy again due to any error in setting up
-	 * origin tracking.
-	 */
-	originid = replorigin_by_name(originname, true);
-	if (!OidIsValid(originid))
+	if (!MyLogicalRepWorker->created_slot)
 	{
+		walrcv_create_slot(LogRepWorkerWalRcvConn,
+						   slotname, false /* permanent */ , false /* two_phase */ ,
+						   CRS_USE_SNAPSHOT, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: created replication slot %s for subscription %u",
+						slotname, MyLogicalRepWorker->subid)));
+
 		/*
-		 * Origin tracking does not exist, so create it now.
-		 *
-		 * Then advance to the LSN got from walrcv_create_slot. This is WAL
-		 * logged for the purpose of recovery. Locks are to prevent the
-		 * replication origin from vanishing while advancing.
+		 * Remember that we created the slot so that we will not try to create
+		 * it again.
 		 */
-		originid = replorigin_create(originname);
-
-		LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
-		replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
-						   true /* go backward */ , true /* WAL log */ );
-		UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+		SpinLockAcquire(&MyLogicalRepWorker->relmutex);
+		MyLogicalRepWorker->created_slot = true;
+		SpinLockRelease(&MyLogicalRepWorker->relmutex);
 
-		replorigin_session_setup(originid, 0);
-		replorigin_session_origin = originid;
+		/*
+		 * Setup replication origin tracking. The purpose of doing this before
+		 * the copy is to avoid doing the copy again due to any error in
+		 * setting up origin tracking.
+		 */
+		if (!OidIsValid(originid))
+		{
+			/*
+			 * Origin tracking does not exist, so create it now.
+			 */
+			originid = replorigin_create(originname);
+		}
+		else
+		{
+			/*
+			 * At this point, there shouldn't be any existing replication
+			 * origin with the same name.
+			 */
+			ereport(ERROR,
+					(errcode(ERRCODE_DUPLICATE_OBJECT),
+					 errmsg("replication origin \"%s\" already exists",
+							originname)));
+		}
 	}
 	else
 	{
-		ereport(ERROR,
-				(errcode(ERRCODE_DUPLICATE_OBJECT),
-				 errmsg("replication origin \"%s\" already exists",
-						originname)));
+		/*
+		 * Do not create a new replication slot, reuse the existing one
+		 * instead. Use a new snapshot for the replication slot to ensure that
+		 * tablesync and apply proceses are consistent with each other.
+		 */
+		WalRcvStreamOptions options;
+		int			server_version;
+
+		server_version = walrcv_server_version(LogRepWorkerWalRcvConn);
+		options.proto.logical.proto_version =
+			server_version >= 150000 ? LOGICALREP_PROTO_TWOPHASE_VERSION_NUM :
+			server_version >= 140000 ? LOGICALREP_PROTO_STREAM_VERSION_NUM :
+			LOGICALREP_PROTO_VERSION_NUM;
+		options.proto.logical.publication_names = MySubscription->publications;
+
+		walrcv_slot_snapshot(LogRepWorkerWalRcvConn, slotname, &options, origin_startpos);
+		ereport(DEBUG2,
+				(errmsg("LogicalRepSyncTableStart: reusing replication slot %s for relation %u in subscription %u",
+						slotname, MyLogicalRepWorker->relid,
+						MyLogicalRepWorker->subid)));
 	}
 
+	/*
+	 * Advance to the LSN got from walrcv_create_slot or walrcv_slot_snapshot.
+	 * This is WAL logged for the purpose of recovery. Locks are to prevent
+	 * the replication origin from vanishing while advancing.
+	 *
+	 * Then setup replication origin tracking.
+	 */
+	LockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+	replorigin_advance(originid, *origin_startpos, InvalidXLogRecPtr,
+					   true /* go backward */ , true /* WAL log */ );
+	UnlockRelationOid(ReplicationOriginRelationId, RowExclusiveLock);
+
+	replorigin_session_setup(originid, 0);
+	replorigin_session_origin = originid;
+
 	/*
 	 * Make sure that the copy command runs as the table owner, unless the
 	 * user has opted out of that behaviour.
@@ -1537,12 +1722,18 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos, int worker_slot)
 	 * Update the persisted state to indicate the COPY phase is done; make it
 	 * visible to others.
 	 */
-	UpdateSubscriptionRelState(MyLogicalRepWorker->subid,
-							   MyLogicalRepWorker->relid,
-							   SUBREL_STATE_FINISHEDCOPY,
-							   MyLogicalRepWorker->relstate_lsn);
+	UpdateSubscriptionRel(MyLogicalRepWorker->subid,
+						  MyLogicalRepWorker->relid,
+						  SUBREL_STATE_FINISHEDCOPY,
+						  MyLogicalRepWorker->relstate_lsn,
+						  slotname,
+						  originname);
 
 	CommitTransactionCommand();
+	ereport(DEBUG2,
+			(errmsg("LogicalRepSyncTableStart: updated originname: %s, slotname: %s, state: %c for relation \"%u\" in subscription \"%u\".",
+					originname, slotname, SUBREL_STATE_FINISHEDCOPY,
+					MyLogicalRepWorker->relid, MyLogicalRepWorker->subid)));
 
 copy_table_done:
 
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index ca663445c1..8966e02b77 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -466,8 +466,16 @@ ReplicationOriginNameForLogicalRep(Oid suboid, Oid relid,
 {
 	if (OidIsValid(relid))
 	{
-		/* Replication origin name for tablesync workers. */
-		snprintf(originname, szoriginname, "pg_%u_%u", suboid, relid);
+		bool		is_null = true;
+
+		/*
+		 * Replication origin name for tablesync workers. First, look into the
+		 * catalog. If originname does not exist, then use the default name.
+		 */
+		GetSubscriptionRelOrigin(suboid, relid,
+								 originname, &is_null);
+		if (is_null)
+			snprintf(originname, szoriginname, "pg_%u_%lld", suboid, (long long) MyLogicalRepWorker->rep_slot_id);
 	}
 	else
 	{
@@ -4504,6 +4512,9 @@ start_table_sync(XLogRecPtr *origin_startpos,
 
 	/* allocate slot name in long-lived context */
 	*myslotname = MemoryContextStrdup(ApplyContext, syncslotname);
+
+	/* Keep the replication slot name used for this sync. */
+	MyLogicalRepWorker->slot_name = *myslotname;
 	pfree(syncslotname);
 }
 
@@ -4557,13 +4568,25 @@ run_tablesync_worker(WalRcvStreamOptions *options,
 {
 	MyLogicalRepWorker->is_sync_completed = false;
 
+	/*
+	 * If it's already connected to the publisher, end streaming before using
+	 * the same connection for another iteration
+	 */
+	if (LogRepWorkerWalRcvConn != NULL)
+	{
+		TimeLineID tli;
+		walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+	}
+
 	/* Start table synchronization. */
 	start_table_sync(origin_startpos, &slotname, worker_slot);
 
+	StartTransactionCommand();
 	ReplicationOriginNameForLogicalRep(MySubscription->oid,
 									   MyLogicalRepWorker->relid,
 									   originname,
 									   originname_size);
+	CommitTransactionCommand();
 
 	set_apply_error_context_origin(originname);
 
@@ -4603,11 +4626,10 @@ run_apply_worker(WalRcvStreamOptions *options,
 				(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
 					errmsg("subscription has no replication slot set")));
 
-	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
-									   originname, originname_size);
-
 	/* Setup replication origin tracking. */
 	StartTransactionCommand();
+	ReplicationOriginNameForLogicalRep(MySubscription->oid, InvalidOid,
+									   originname, originname_size);
 	originid = replorigin_by_name(originname, true);
 	if (!OidIsValid(originid))
 		originid = replorigin_create(originname);
@@ -4924,7 +4946,32 @@ TablesyncWorkerMain(Datum main_arg)
 			}
 
 			if (!is_table_found)
+			{
+				TimeLineID	tli;
+
+				/*
+				 * It is important to give an error if we are unable to drop the
+				 * slot, otherwise, it won't be dropped till the corresponding
+				 * subscription is dropped. So passing missing_ok = false.
+				 */
+				if (MyLogicalRepWorker->created_slot)
+				{
+					walrcv_endstreaming(LogRepWorkerWalRcvConn, &tli);
+					ReplicationSlotDropAtPubNode(LogRepWorkerWalRcvConn, MyLogicalRepWorker->slot_name, false);
+				}
+
+				/*
+				 * Drop replication origin before exiting.
+				 *
+				 * There is a chance that the user is concurrently performing refresh
+				 * for the subscription where we remove the table state and its origin
+				 * or the apply worker would have removed this origin. So passing
+				 * missing_ok = true.
+				 */
+				replorigin_drop_by_name(originname, true, false);
+
 				break;
+			}
 		}
 	}
 
diff --git a/src/include/catalog/pg_subscription.h b/src/include/catalog/pg_subscription.h
index 1d40eebc78..7e13f59847 100644
--- a/src/include/catalog/pg_subscription.h
+++ b/src/include/catalog/pg_subscription.h
@@ -108,6 +108,9 @@ CATALOG(pg_subscription,6100,SubscriptionRelationId) BKI_SHARED_RELATION BKI_ROW
 
 	/* Only publish data originating from the specified origin */
 	text		suborigin BKI_DEFAULT(LOGICALREP_ORIGIN_ANY);
+
+	/* The last used ID to create a replication slot for tablesync */
+	int64		sublastusedid BKI_DEFAULT(0);
 #endif
 } FormData_pg_subscription;
 
@@ -144,6 +147,8 @@ typedef struct Subscription
 	List	   *publications;	/* List of publication names to subscribe to */
 	char	   *origin;			/* Only publish data originating from the
 								 * specified origin */
+	int64		lastusedid;		/* Last used unique ID to create replication
+								 * slots in tablesync */
 } Subscription;
 
 /* Disallow streaming in-progress transactions. */
@@ -164,6 +169,7 @@ typedef struct Subscription
 extern Subscription *GetSubscription(Oid subid, bool missing_ok);
 extern void FreeSubscription(Subscription *sub);
 extern void DisableSubscription(Oid subid);
+extern void UpdateSubscriptionLastSlotId(Oid subid, int64 lastusedid);
 
 extern int	CountDBSubscriptions(Oid dbid);
 
diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h
index 60a2bcca23..185164d75e 100644
--- a/src/include/catalog/pg_subscription_rel.h
+++ b/src/include/catalog/pg_subscription_rel.h
@@ -44,6 +44,12 @@ CATALOG(pg_subscription_rel,6102,SubscriptionRelRelationId)
 											 * used for synchronization
 											 * coordination, or NULL if not
 											 * valid */
+	NameData	srrelslotname BKI_FORCE_NULL;	/* name of the replication
+												 * slot for relation in
+												 * subscription */
+	NameData	srreloriginname BKI_FORCE_NULL; /* origin name for relation in
+												 * subscription */
+
 #endif
 } FormData_pg_subscription_rel;
 
@@ -81,10 +87,16 @@ typedef struct SubscriptionRelState
 } SubscriptionRelState;
 
 extern void AddSubscriptionRelState(Oid subid, Oid relid, char state,
-									XLogRecPtr sublsn);
+									XLogRecPtr sublsn, char *relslotname, char *reloriginname);
 extern void UpdateSubscriptionRelState(Oid subid, Oid relid, char state,
 									   XLogRecPtr sublsn);
+extern void UpdateSubscriptionRel(Oid subid, Oid relid, char state,
+								  XLogRecPtr sublsn, char *relslotname, char *reloriginname);
+
 extern char GetSubscriptionRelState(Oid subid, Oid relid, XLogRecPtr *sublsn);
+extern void GetSubscriptionRelReplicationSlot(Oid subid, Oid relid, char *slotname);
+extern void GetSubscriptionRelOrigin(Oid subid, Oid relid, char *reloriginname, bool *isnull);
+
 extern void RemoveSubscriptionRel(Oid subid, Oid relid);
 
 extern bool HasSubscriptionRelations(Oid subid);
diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h
index a8a89dc784..31b2c41893 100644
--- a/src/include/replication/slot.h
+++ b/src/include/replication/slot.h
@@ -237,8 +237,9 @@ extern bool InvalidateObsoleteReplicationSlots(ReplicationSlotInvalidationCause
 extern ReplicationSlot *SearchNamedReplicationSlot(const char *name, bool need_lock);
 extern int	ReplicationSlotIndex(ReplicationSlot *slot);
 extern bool ReplicationSlotName(int index, Name name);
-extern void ReplicationSlotNameForTablesync(Oid suboid, Oid relid, char *syncslotname, Size szslot);
+extern void ReplicationSlotNameForTablesync(Oid suboid, int64 slotid, char *syncslotname, Size szslot);
 extern void ReplicationSlotDropAtPubNode(WalReceiverConn *wrconn, char *slotname, bool missing_ok);
+extern List *GetReplicationSlotNamesBySubId(WalReceiverConn *wrconn, Oid subid, bool missing_ok);
 
 extern void StartupReplicationSlots(void);
 extern void CheckPointReplicationSlots(void);
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index af6fd339f7..97c7d2aa50 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -35,6 +35,23 @@ typedef struct LogicalRepWorker
 	/* Indicates if this slot is used or free. */
 	bool		in_use;
 
+	/*
+	 * Indicates if the sync worker created a replication slot for itself
+	 * in any point of its lifetime.
+	 * False means that the worker has not created a slot yet, and has been
+	 * reusing replication slots created by other workers so far.
+	 */
+	bool		created_slot;
+
+	/*
+	 * Unique identifier for replication slot to be created by tablesnync
+	 * workers, if needed.
+	 */
+	int64		rep_slot_id;
+
+	/* Replication slot name used by the worker. */
+	char	   *slot_name;
+
 	/* Increased every time the slot is taken by new worker. */
 	uint16		generation;
 
@@ -239,7 +256,8 @@ extern LogicalRepWorker *logicalrep_worker_find(Oid subid, Oid relid,
 extern List *logicalrep_workers_find(Oid subid, bool only_running);
 extern bool logicalrep_worker_launch(Oid dbid, Oid subid, const char *subname,
 									 Oid userid, Oid relid,
-									 dsm_handle subworker_dsm);
+									 dsm_handle subworker_dsm,
+									 int64 slotid);
 extern void logicalrep_worker_stop(Oid subid, Oid relid);
 extern void logicalrep_pa_worker_stop(ParallelApplyWorkerInfo *winfo);
 extern void logicalrep_worker_wakeup(Oid subid, Oid relid);
@@ -336,4 +354,7 @@ am_parallel_apply_worker(void)
 	return isParallelApplyWorker(MyLogicalRepWorker);
 }
 
+/* Invalid identifier to be used for naming replication slots */
+#define InvalidRepSlotId	0
+
 #endif							/* WORKER_INTERNAL_H */
-- 
2.27.0